{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>3000</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>5465</td>\n",
       "      <td>2732.5</td>\n",
       "      <td>1821.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.0</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3275</td>\n",
       "      <td>1637.5</td>\n",
       "      <td>1637.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3350</td>\n",
       "      <td>1675.0</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 228 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.5         3   3000           1200.0      750.000000       -1.5   \n",
       "1        1.0         2   5465           2732.5     1821.666667       -1.0   \n",
       "2        1.0         1   2850           1425.0     1425.000000        0.0   \n",
       "3        1.0         1   3275           1637.5     1637.500000        0.0   \n",
       "4        1.0         4   3350           1675.0      670.000000       -3.0   \n",
       "\n",
       "   room_num  Year  Month  Day       ...        walk  walls  war  washer  \\\n",
       "0       4.5  2016      6   24       ...           0      0    0       0   \n",
       "1       3.0  2016      6   12       ...           0      0    0       0   \n",
       "2       2.0  2016      4   17       ...           0      0    0       0   \n",
       "3       2.0  2016      4   18       ...           0      0    0       0   \n",
       "4       5.0  2016      4   28       ...           0      0    1       0   \n",
       "\n",
       "   water  wheelchair  wifi  windows  work  interest_level  \n",
       "0      0           0     0        0     0               1  \n",
       "1      0           0     0        0     0               2  \n",
       "2      0           0     0        0     0               0  \n",
       "3      0           0     0        0     0               2  \n",
       "4      0           0     0        0     0               2  \n",
       "\n",
       "[5 rows x 228 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#导入必要工具包\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "#从文件中读取数据集\n",
    "train = pd.read_csv(\"./data/RentListingInquries_FE_train.csv\")\n",
    "#打印前5行数据\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(49352, 228)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#打印行列\n",
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 49352 entries, 0 to 49351\n",
      "Columns: 228 entries, bathrooms to interest_level\n",
      "dtypes: float64(9), int64(219)\n",
      "memory usage: 85.8 MB\n"
     ]
    }
   ],
   "source": [
    "#打印数据信息\n",
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:2026: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "#数据太大了，为了能尽快完成作业，随机取其中一部分数据进行测试\n",
    "train_y = train['interest_level']\n",
    "train_X = train.drop(['interest_level'], axis = 1)\n",
    "\n",
    "#随机抽取一定的数据\n",
    "X_train,X_test,y_train,y_test = train_test_split(train_X, train_y, train_size = 0.02, random_state = 33)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(987, 227)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "#xgboost 导入必要的工具包\n",
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEKCAYAAADaa8itAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHV5JREFUeJzt3Xu4XFWd5vHvS8JtEEzQyJNOwECbtsVbhANEaS8gQqCnTfA2MA4EZTqtAwpqK8HRjoLYaotM4wVND5HEsY0RQaJGMY2Ao80twXAJgc4REI6kSTDc0dDg23/sdUJ5qHPOzqEqReW8n+epp2r/9lq7fkUezu/Ze629tmwTERHRCtt1OoGIiNh2pKhERETLpKhERETLpKhERETLpKhERETLpKhERETLpKhERETLpKhERETLpKhERETLjO10Alvb85//fE+ZMqXTaUREdJWVK1feZ3vCcO1GXVGZMmUKK1as6HQaERFdRdKv67TL5a+IiGiZthUVSTtJulbSDZJWS/pkiV8g6Q5Jq8prWolL0rmSeiXdKGm/hmPNlrS2vGY3xPeXdFPpc64ktev3RETE8Np5+WsTcKjtRyRtD/xc0o/Kvg/bvnBA+yOBqeV1EHAecJCk3YF5QA9gYKWkpbbvL23mAFcDy4AZwI+IiIiOaNuZiiuPlM3ty2uodfZnAotKv6uBcZImAkcAy21vLIVkOTCj7NvN9lWu1u9fBMxq1++JiIjhtXVMRdIYSauA9VSF4Zqy66xyiescSTuW2CTg7obufSU2VLyvSbxZHnMkrZC0YsOGDc/4d0VERHNtLSq2n7Q9DZgMHCjpZcDpwJ8DBwC7A6eV5s3GQzyCeLM85tvusd0zYcKwM+IiImKEtsrsL9sPAFcAM2yvK5e4NgFfBw4szfqAPRu6TQbuGSY+uUk8IiI6pJ2zvyZIGlc+7wwcBtxaxkIoM7VmATeXLkuB48sssOnAg7bXAZcCh0saL2k8cDhwadn3sKTp5VjHA5e06/dERMTw2jn7ayKwUNIYquK1xPYPJP1U0gSqy1ergPeU9suAo4Be4DHgXQC2N0o6E7iutDvD9sby+b3ABcDOVLO+MvMrIqKDVE2cGj16enqcO+ojnt0O/uLBnU5hm/eL9/1ii9pLWmm7Z7h2uaM+IiJaJkUlIiJaJkUlIiJaJkUlIiJaJkUlIiJaJkUlIiJaJkUlIiJaJkUlIiJaJkUlIiJaJkUlIiJaJkUlIiJaJkUlIiJaJkUlIiJaJkUlIiJaJkUlIiJaJkUlIiJaJkUlIiJaJkUlIiJaJkUlIiJaJkUlIiJaJkUlIiJapm1FRdJOkq6VdIOk1ZI+WeJ7S7pG0lpJ35a0Q4nvWLZ7y/4pDcc6vcRvk3REQ3xGifVKmtuu3xIREfW080xlE3Co7VcC04AZkqYDnwXOsT0VuB84sbQ/Ebjf9ouAc0o7JO0LHAO8FJgBfEXSGEljgC8DRwL7AseWthER0SFtKyquPFI2ty8vA4cCF5b4QmBW+TyzbFP2v1GSSnyx7U227wB6gQPLq9f27bYfBxaXthER0SG1i4qk8ZJeKmkfSbX6lTOKVcB6YDnwK+AB20+UJn3ApPJ5EnA3QNn/IPC8xviAPoPFIyKiQ8YOtVPSc4GTgGOBHYANwE7AHpKuBr5i+/LB+tt+EpgmaRxwMfCSZs36v26QfYPFmxU2N4khaQ4wB2CvvfYaLN2IiHiGhiwqVJehFgGvtf1A4w5J+wPHSdrH9vlDHcT2A5KuAKYD4ySNLWcjk4F7SrM+YE+gT9JY4LnAxoZ4v8Y+g8UHfv98YD5AT09P08ITERHP3JCXsWy/yfY3BhaUsm+l7VMHKyiSJpQzFCTtDBwGrAEuB95Wms0GLimfl5Ztyv6f2naJH1Nmh+0NTAWuBa4DppbZZDtQDeYvrfvDIyKi9YY7U9lM0iTghY19bP9siC4TgYVlltZ2wBLbP5B0C7BY0qeAXwL9Rel84BuSeqnOUI4p37Fa0hLgFuAJ4KRyWQ1JJwOXAmOABbZX1/09ERHRerWKiqTPAv+N6g/7kyVsYNCiYvtG4FVN4rdTzdwaGP898PZBjnUWcFaT+DJg2fC/ICIitoa6ZyqzgBfb3tTOZCIiorvVnVJ8O9V9JhEREYOqe6byGLBK0mVUd8oDYPv9bckqIiK6Ut2ispTMrIqIiGHUKiq2Fw7fKiIiRrvh7qhfYvsdkm6iyd3qtl/RtswiIqLrDHemckp5/6/tTiQiIrrfkEXF9rry/uutk05ERHSzuqsNT5d0naRHJD0u6UlJD7U7uYiI6C5171P5EtVKxWuBnYH/CXyxXUlFRER3qr32l+1eSWPKultfl/SvbcwrIiK6UO2bH8tKwKskfQ5YB+zSvrQiIqIb1b38dVxpezLwKNVzTN7arqQiIqI7DXumUpauP8v2/wB+D3yy7VlFRERXGvZMpYyhTCiXvyIiIgZVd0zlTuAXkpZSXf4CwPYX2pFURER0p7pF5Z7y2g7YtcTyrPeIiPgjdYvKLba/0xiQ1PQpjRERMXrVnf11es1YRESMYsOtUnwkcBQwSdK5Dbt2A55oZ2IREdF9hrv8dQ+wEnhzee/3MPCBdiUVERHdacjLX7ZvsH0B8CLbCxteF9m+f6i+kvaUdLmkNZJWSzqlxD8h6TeSVpXXUQ19TpfUK+k2SUc0xGeUWK+kuQ3xvSVdI2mtpG9n2nNERGcNWVQkfV/SXw2ybx9JZ0h69yDdnwA+ZPslwHTgJEn7ln3n2J5WXsvK8fYFjgFeCswAviJpTLn58svAkcC+wLENx/lsOdZU4H7gxJq/OyIi2mC4gfq/Bl4L3FqWvl8m6aeS7gC+Bqy0vaBZR9vrbF9fPj8MrAEmDfFdM4HFtjfZvgPoBQ4sr17bt9t+HFgMzJQk4FDgwtJ/ITCrxm+OiIg2Ge4hXf8OfAT4iKQpwETgd8C/2X6s7peUvq8CrgEOBk6WdDywgups5n6qgnN1Q7c+nipCdw+IHwQ8D3jA9hNN2kdERAfUfUjXLsBdtq8CHgMOk7R9zb7PAb4LnGr7IeA84E+BaVSrHZ/d37RJd48g3iyHOZJWSFqxYcOGOmlHRMQI1L1P5WfATpImAZcB7wIuGK5TKTzfBb5p+yIA2/faftL2H4B/orq8BdWZxp4N3SdTzT4bLH4fME7S2AHxp7E933aP7Z4JEybU+LkRETESdYuKyuWutwBftH001aD54B2qMY/zgTWNa4RJmtjQ7Gjg5vJ5KXCMpB0l7Q1MBa4FrgOmlpleO1AN5i+1beBy4G2l/2zgkpq/JyIi2qDuMi2S9GrgnTw1w2q4vgdTPYflJkmrSuyjVLO3plFdqroT+BsA26slLQFuoZo5dlJZIRlJJwOXAmOABbZXl+OdBiyW9Cngl1RFLCIiOqRuUTmValmWi8sf/32ozhIGZfvnNB/3WDZEn7OAs5rElzXrZ/t2nrp8FhERHVarqNi+EriyDNj3/zF/fzsTi4iI7lN39terJd1Cda8Jkl4p6SttzSwiIrpO3YH6/wMcAfwWquVbgNe1K6mIiOhOdYsKtu8eEHqyxblERESXqztQf7ek1wAu03rfT7kUFhER0a/umcp7gJOolkHpo7ob/qR2JRUREd2p7uyv+6juUYmIiBhU3dlfCyWNa9geL6np6sQRETF61b389QrbD/RvlFWFX9WelCIiolvVLSrbSRrfvyFpd+oP8kdExChRtzCcDfyrpP4HYr2dJsupRETE6FZ3oH6RpJXAIVTreb3F9i1tzSwiIrrOllzCupXqOfBjASTtZfuutmQVERFdqVZRkfQ+YB5wL9Wd9KJauv4V7UstIiK6Td0zlVOAF9v+bTuTiYiI7lZ39tfdwIPtTCQiIrpf3TOV24ErJP0Q2NQfbHxMcERERN2icld57VBeERERT1N3SvEnASTtYvvR9qYUERHdKk9+jIiIlsmTHyMiomXa9uRHSXtKulzSGkmrJZ1S4rtLWi5pbXkfX+KSdK6kXkk3Stqv4VizS/u1kmY3xPeXdFPpc64k1f09ERHRerWnFDc++VHS3zL8kx+fAD5k+yXAdOAkSfsCc4HLbE8FLivbAEcCU8trDnAebF68ch5wEHAgMK9hccvzStv+fjNq/p6IiGiDtj350fY629eXzw9TFaFJwExgYWm2EJhVPs8EFrlyNTBO0kSqy27LbW8sS+4vB2aUfbvZvsq2gUUNx4qIiA4YdvaXpDHAcbZH/ORHSVOonr9yDbCH7XVQFR5JLyjNJlHdZNmvr8SGivc1iUdERIcMe6Zi+0mqs4gRkfQc4LvAqbYfGqpps68fQbxZDnMkrZC0YsOGDcOlHBERI1T38tcvJH1J0msl7df/Gq6TpO2pCso3bV9UwveWS1eU9/Ul3gfs2dB9MnDPMPHJTeJPY3u+7R7bPRMmTBgu7YiIGKG6ReU1wEuBM6ge2HU28PmhOpSZWOcDawYs57IU6J/BNRu4pCF+fJkFNh14sFwmuxQ4XNL4MkB/OHBp2fewpOnlu45vOFZERHRAnTGV7YDzbC/ZwmMfDBwH3CRpVYl9FPgMsETSiVRLv7y97FsGHAX0Ao8B7wKwvVHSmcB1pd0ZtjeWz+8FLgB2Bn5UXhER0SHDFhXbf5B0MrBFRcX2z2k+7gHwxibtzSAzymwvABY0ia8AXrYleUVERPvUvfy1XNLflhsad+9/tTWziIjoOnVXKX53eW88kzCwT2vTiYiIblZ3leK9251IRER0v7rPqD++Wdz2otamExER3azu5a8DGj7vRDXQfj3V0igRERFA/ctf72vclvRc4BttySgiIrpW7aXvB3iMalXgiIiIzeqOqXyfp9bV2g7Yly28byUiIrZ9dcdUGpdkeQL4te2+wRpHRMToVLeo3AWss/17AEk7S5pi+862ZRYREV2n7pjKd4A/NGw/WWIRERGb1S0qY20/3r9RPu/QnpQiIqJb1S0qGyS9uX9D0kzgvvakFBER3arumMp7gG9K+lLZ7qN6fklERMRmdW9+/BUwvTwaWLYfbm9aERHRjWpd/pL0aUnjbD9i++HyFMZPtTu5iIjoLnXHVI60/UD/hu37qZ7SGBERsVndojJG0o79G5J2BnYcon1ERIxCdQfq/x9wmaSvUy3X8m5gYduyioiIrlR3oP5zkm4EDiuhM21f2r60IiKiG9U9UwH4JbA91ZnKL9uTTkREdLO6s7/eAVwLvA14B3CNpLe1M7GIiOg+dQfq/zdwgO3Zto8HDgQ+PlQHSQskrZd0c0PsE5J+I2lVeR3VsO90Sb2SbpN0REN8Ron1SprbEN9b0jWS1kr6tqQsGxMR0WF1i8p2ttc3bP+2Rt8LgBlN4ufYnlZeywAk7QscA7y09PmKpDGSxgBfBo6keobLsaUtwGfLsaYC9wMn1vwtERHRJnWLyo8lXSrpBEknAD8Elg3VwfbPgI01jz8TWGx7k+07gF6qs6EDgV7bt5dFLBcDMyUJOBS4sPRfCMyq+V0REdEmtYqK7Q8DXwNeAbwSmG/7tBF+58mSbiyXx8aX2CTg7oY2fSU2WPx5wAO2nxgQb0rSHEkrJK3YsGHDCNOOiIjh1H5Gve2LbH/Q9gdsXzzC7zsP+FNgGrAOOLvE1ewrRxBvyvZ82z22eyZMmLBlGUdERG1bMqX4GbN9b/9nSf8E/KBs9gF7NjSdDNxTPjeL3weMkzS2nK00to+IiA6pfabSCpImNmweDfTPDFsKHCNpR0l7A1OppjBfB0wtM712oBrMX2rbwOVUU5wBZgOXbI3fEBERgxuyqEi6rLx/dksPLOlbwFXAiyX1SToR+Jykm8rd+YcAHwCwvRpYAtwC/Bg4yfaT5SzkZOBSYA2wpLQFOA34oKReqjGW87c0x4iIaK3hLn9NlPR64M2SFjNgLMP29YN1tH1sk/Cgf/htnwWc1SS+jCYzzWzfTjU7LCIiniWGKyp/B8ylGrP4woB9pprWGxERAQxTVGxfCFwo6eO2z9xKOUVERJequ0rxmZLeDLyuhK6w/YOh+kRExOhTd0HJvwdOoRpIvwU4pcQiIiI2q3ufyl8C02z/AUDSQqrl709vV2IREdF9tuQ+lXENn5/b6kQiIqL71T1T+Xvgl5Iup5pW/DpylhIREQPUHaj/lqQrgAOoispptv+9nYlFRET3qb32l+11VMupRERENLVV1/6KiIhtW4pKRES0zLBFRdJ2jc+Zj4iIGMywRaXcm3KDpL22Qj4REdHF6g7UTwRWS7oWeLQ/aPvNbckqIiK6Ut2i8sm2ZhEREduEuvepXCnphcBU2/8i6b8AY9qbWkREdJu6C0r+NXAh8LUSmgR8r11JRUREd6p7+eskqqcsXgNge62kF7Qtq4hn6K4zXt7pFEaFvf7upk6nEM8yde9T2WT78f4NSWOpnvwYERGxWd2icqWkjwI7S3oT8B3g++1LKyIiulHdojIX2ADcBPwNsAz4WLuSioiI7lSrqJQbIBcCZ1JNL15oe8jLX5IWSFrfeDe+pN0lLZe0tryPL3FJOldSr6QbJe3X0Gd2ab9W0uyG+P6Sbip9zpWkLfvpERHRanVnf/0l8CvgXOBLQK+kI4fpdgEwY0BsLnCZ7anAZWUb4EhgannNAc4r37s7MA84iGqiwLz+QlTazGnoN/C7IiJiK6t7+ets4BDbb7D9euAQ4JyhOtj+GbBxQHgm1RkP5X1WQ3yRK1cD4yRNBI4AltveaPt+YDkwo+zbzfZV5YxpUcOxIiKiQ+oWlfW2exu2bwfWj+D79ijPZel/Pkv/tORJwN0N7fpKbKh4X5N4U5LmSFohacWGDRtGkHZERNQx5H0qkt5SPq6WtAxYQjWV+O3AdS3Mo9l4iEcQb8r2fGA+QE9PT6ZCR0S0yXA3P/5Vw+d7gdeXzxuA8U9vPqx7JU20va5cwuo/2+kD9mxoNxm4p8TfMCB+RYlPbtI+IiI6aMiiYvtdLf6+pcBs4DPl/ZKG+MmSFlMNyj9YCs+lwKcbBucPB063vVHSw5KmU93lfzzwxRbnGhERW6jWMi2S9gbeB0xp7DPU0veSvkV1lvF8SX1Us7g+AyyRdCJwF9VlNKjuezkK6AUeA95Vjr9R0pk8dantDNv9g//vpZphtjPwo/KKiIgOqrv21/eA86nuov9DnQ62jx1k1xubtDXV+mLNjrMAWNAkvgJ4WZ1cIiJi66hbVH5v+9y2ZhIREV2vblH5R0nzgJ8Am/qDtq9vS1YREdGV6haVlwPHAYfy1OUvl+2IiAigflE5Gtincfn7iIiIgereUX8DMK6diURERPere6ayB3CrpOv44zGVQacUR0TE6FO3qMxraxYREbFNqFVUbF/Z7kQiIqL71b2j/mGeWrBxB2B74FHbu7UrsYiI6D51z1R2bdyWNIvqoVkRERGb1Z399Udsf4/coxIREQPUvfz1lobN7YAehnh+SUREjE51Z381PlflCeBOqkcAR0REbFZ3TKXVz1WJiIht0HCPE/67IXbb9pktziciIrrYcGcqjzaJ7QKcCDwPSFGJiIjNhnuc8Nn9nyXtCpxC9VTGxcDZg/WLiIjRadgxFUm7Ax8E3gksBPazfX+7E4uIiO4z3JjKPwBvAeYDL7f9yFbJKiIiutJwNz9+CPgT4GPAPZIeKq+HJT3U/vQiIqKbDDemMqI77iMiYnTqSNGQdKekmyStkrSixHaXtFzS2vI+vsQl6VxJvZJulLRfw3Fml/ZrJc3uxG+JiIindPJM5BDb02z3lO25wGW2pwKXlW2AI4Gp5TUHOA82TyCYBxxEtbjlvP5CFBERnfFsurw1k2p2GeV9VkN8kStXA+MkTQSOAJbb3lhmoy0HZmztpCMi4imdKioGfiJppaQ5JbaH7XUA5f0FJT4JuLuhb1+JDRaPiIgOqbugZKsdbPseSS8Alku6dYi2ahLzEPGnH6AqXHMA9tprry3NNSIiaurImYrte8r7euBiqjGRe8tlLcr7+tK8D9izoftk4J4h4s2+b77tHts9EyZMaOVPiYiIBlu9qEjapSz5gqRdgMOBm4GlQP8MrtnAJeXzUuD4MgtsOvBguTx2KXC4pPFlgP7wEouIiA7pxOWvPYCLJfV//z/b/rGk64Alkk4E7gLeXtovA44CeoHHqNYew/ZGSWcC15V2Z9jeuPV+RkREDLTVi4rt24FXNon/Fnhjk7iBkwY51gJgQatzjIiIkXk2TSmOiIgul6ISEREt06kpxV1h/w8v6nQK27yV/3B8p1OIiBbKmUpERLRMikpERLRMikpERLRMikpERLRMikpERLRMikpERLRMikpERLRMikpERLRMikpERLRMikpERLRMikpERLRMikpERLRMikpERLRMikpERLRMikpERLRMikpERLRMikpERLRMikpERLRM1xcVSTMk3SapV9LcTucTETGadXVRkTQG+DJwJLAvcKykfTubVUTE6NXVRQU4EOi1fbvtx4HFwMwO5xQRMWp1e1GZBNzdsN1XYhER0QFjO53AM6QmMT+tkTQHmFM2H5F0W1uz6qznA/d1Oom69PnZnU7h2aSr/u0AmNfsf8FRq6v+/fT+Lf63e2GdRt1eVPqAPRu2JwP3DGxkez4wf2sl1UmSVtju6XQeseXyb9fd8u9X6fbLX9cBUyXtLWkH4BhgaYdziogYtbr6TMX2E5JOBi4FxgALbK/ucFoREaNWVxcVANvLgGWdzuNZZFRc5ttG5d+uu+XfD5D9tHHtiIiIEen2MZWIiHgWSVHZRmS5mu4laYGk9ZJu7nQusWUk7SnpcklrJK2WdEqnc+q0XP7aBpTlav4NeBPVNOvrgGNt39LRxKIWSa8DHgEW2X5Zp/OJ+iRNBCbavl7SrsBKYNZo/n8vZyrbhixX08Vs/wzY2Ok8YsvZXmf7+vL5YWANo3xVjxSVbUOWq4noMElTgFcB13Q2k85KUdk21FquJiLaQ9JzgO8Cp9p+qNP5dFKKyrah1nI1EdF6kranKijftH1Rp/PptBSVbUOWq4noAEkCzgfW2P5Cp/N5NkhR2QbYfgLoX65mDbAky9V0D0nfAq4CXiypT9KJnc4pajsYOA44VNKq8jqq00l1UqYUR0REy+RMJSIiWiZFJSIiWiZFJSIiWiZFJSIiWiZFJSIiWiZFJWIASeMk/a+t8D1vkPSaAbFTJR1fPp8g6U9GcNz39B9jiDYvl3TBlh47YjgpKhFPNw6oXVRUGcn/S28ANhcVSWOBdwP/XEInAE2LSlmZuinbX7W9aKgvtn0TMFnSXluWcsTQcp9KxACS+ld5vg24HHgFMB7YHviY7UvK4oE/KvtfDcwCDgNOo1oiZy2wyfbJkiYAXwX6/4CfCvwGuBp4EtgAvA/YGfjvtk+Q9DbggtLud+U71gALgMOBLwG7AnOAHYBe4Djbj0n6BPCI7c9LuoJqgcNDqIrlibb/f/mdpwA72v5c6/7rxWiXM5WIp5sL/Mr2NODDwNG296P6w3x2WZoD4MVUz0B5FfAfwMeB6VTPtfnzhuP9I3CO7QOAtwL/1/adVIXmHNvTyh/6g6mex4HtC4EVwDvL/t+VY/3e9l/YXgxcZPsA26+kKjiD3Yk/1vaBVMVsXkN8BfDakfwHihjM2E4nEPEsJ+DT5UFaf6B6pMAeZd+vbV9dPh8IXGl7I4Ck7wB/VvYdBuz7VC1it/JAp4EmUhWHoXy74fPLJH2K6gzkOVTL9DTTv8jhSmBKQ3w9g1xeixipFJWIob0TmADsb/s/JN0J7FT2PdrQrtnjB/ptB7y64Wyj6qCndfldw7EH0/idF1A9ZfAGSSdQjdE0s6m8P8kf/z+/U/nOiJbJ5a+Ip3uYarwC4LnA+lJQDgFeOEifa4HXSxpfBtzf2rDvJ1QLfgIgaVqT74HqLOVFg+TRzK7AurL0+juH/klN/Rlw8wj6RQwqRSViANu/BX4h6WZgGtAjaQXVH+5bB+nzG+DTVIPi/wLcAjxYdr+/HONGSbcA7ynx7wNHl5VtX0s18P+6hsNeAHy17N+5ydd+vHzf8sHyGsYhwA9H0C9iUJn9FdEikp5j+5FypnIxsMD2xVt4jIuBj9he25Ykn/qeHYErgb8oj06IaImcqUS0zickraK6pHQH8L0RHGMu1YB9u+0FzE1BiVbLmUpERLRMzlQiIqJlUlQiIqJlUlQiIqJlUlQiIqJlUlQiIqJlUlQiIqJl/hOicyzqE3kghQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#查看输出结果的分布情况\n",
    "sns.countplot(train.interest_level);\n",
    "pyplot.xlabel('target(train)');\n",
    "pyplot.ylabel('Number of occurrences(train)');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAELCAYAAAAspXpuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGahJREFUeJzt3Xu0nVV57/HvDxDxHqiBk3IxqKnK8IIaLEq9AHpOQQWlYHVYocho7KgiHKuCHi9VHGpbFW8VTwQFPN4QRUDxioCntiIJeOGiEhmYpCCEqyCiAs/5Y811WIS9st8d9spa2fl+xlhjve985/uuZ2ePrGfPd853zlQVkiRNZbNxByBJmlwmCUnSUCYJSdJQJglJ0lAmCUnSUCYJSdJQJglJ0lAmCUnSUCYJSdJQW4w7gPvq4Q9/eC1cuHDcYUjSRmX58uXXVdX86ept9Eli4cKFLFu2bNxhSNJGJcmvutTzdpMkaSiThCRpKJOEJGkok4QkaSiThCRpKJOEJGkok4QkaSiThCRpKJOEJGmojf6Ja0mTb4+P7DHuEDYJ3z/8+7N+TVsSkqShTBKSpKFMEpKkoUwSkqShTBKSpKFMEpKkoUwSkqShTBKSpKFMEpKkoUwSkqShTBKSpKFMEpKkoUaeJJLMS3Jqkp8luSzJ05Nsk+TbSS5v71u3ukny4SQrkvwkyVNGHZ8kabgN0ZL4EPCNqnos8CTgMuBo4OyqWgSc3fYB9gEWtdcS4LgNEJ8kaYiRJokkDwWeBZwAUFV/qKqbgP2Bk1q1k4AXte39gZOr5wfAvCQLRhmjJGm4UbckHgmsAT6V5KIkxyd5ELBdVV0N0N63bfW3B1YNnL+6lUmSxmDUSWIL4CnAcVX1ZOC33H1raSqZoqzuVSlZkmRZkmVr1qyZnUglSfcy6iSxGlhdVee3/VPpJY1r+reR2vu1A/V3HDh/B+CqtS9aVUuranFVLZ4/f/7IgpekTd1Ik0RV/RpYleQxrWhv4FLgDOCQVnYIcHrbPgM4uI1y2h24uX9bSpK04c1ojevWn3B7Vd05g9MOBz6TZEvgCuBQesnplCSHASuBg1rds4B9gRXAba2uJGlM1pkkkmwGvBR4ObAb8Hvg/knW0PtCX1pVl6/rGlX1I2DxFIf2nqJuAa/uFrokadSmu910DvAo4E3Af6uqHatqW+CZwA+A9yb5mxHHKEkak+luNz23qv64dmFV3QB8CfhSkvuNJDJJ0titM0kMJogkmwPbDZ5TVSunSiKSpLmhU8d1ksOBtwPXAHe14gKeOKK4JEkToOvopiOAx1TV9aMMRpI0Wbo+J7EKuHmUgUiSJk/XlsQVwLlJvkZvGCwAVfWBkUQlSZoIXZPEyvbasr0kSZuATkmiqt4x6kAkSZNnuieuP1hVRyY5kylmY62q/UYWmSRp7KZrSXy6vb9v1IFIkibPdA/TLW/v522YcCRJk6Trw3SLgPcAuwBb9cur6pEjikuSNAG6PifxKeA44A5gT+Bk7r4VJUmao7omiQdU1dlAqupXVfVPwF6jC0uSNAm6Pidxe1tb4vIkrwH+C9h2dGFJkiZB15bEkcADgdcCTwX+hruXH5UkzVHTtiTaFOEvqao3ALfikqKStMmYtiXR1rN+apJsgHgkSROka5/ERcDpSb4I/LZfWFVfHklUkqSJ0DVJbANczz1HNBVgkpCkOaxrkji+qr4/WJBkjxHEI0maIF1HN32kY5kkaQ6ZbhbYpwPPAOYned3AoYcCm3f5gCRXArcAdwJ3VNXiJNsAXwAWAlfSGz11Y+sc/xCwL3Ab8LdVdeFMfiBJ0uyZriWxJfBgesnkIQOv3wAHzuBz9qyqXatqcds/Gji7qhYBZ7d9gH2ARe21hN5UIJKkMZluFtjzgPOSnFhVvxpWL8lHqurwGXzu/sBz2vZJwLnAUa385Koq4AdJ5iVZUFVXz+DakqRZ0qlPYl0JollXJ3YB30qyPMmSVrZd/4u/vfen+NgeWDVw7upWdg9JliRZlmTZmjVruvwIkqT10HV0032xR1VdlWRb4NtJfraOulM9sDfVinhLgaUAixcvvtdxSdLs6Dq6ab1V1VXt/VrgNOBpwDVJFgC092tb9dXAjgOn7wBcNeoYJUlTm60kMeWUHUkelOQh/W3gvwMXA2dw9wSBhwCnt+0zgIPTsztws/0RkjQ+XVeme3xVXbyOKh8aUr4dcFqb9mkL4LNV9Y0kFwCnJDkMWAkc1OqfRW/46wp6Q2CdTFCSxqhrn8THk2wJnEjvi/6mwYNVdeJUJ1XVFcCTpii/Hth7ivICXt0xJknSiHUd3fQXwMvp9RcsS/LZJM8baWSSpLHr3CdRVZcDb6H3PMOzgQ8n+VmSA0YVnCRpvDoliSRPTHIscBm9mWBfWFWPa9vHjjA+SdIYde2T+CjwCeDNVfW7fmF7/uEtI4lMkjR2nZJEVT1rHcc+PXvhSJImyTpvNyU5M8kLk9xvimOPTPLOJK8cXXiSpHGariXxd8DrgA8muQFYA2wF7EzvWYaPVtXp6zhfkrQRm24W2F8DbwTemGQhsAD4HfCLqrpt5NFJksaq6+imBwErq+o/6T0J/dypbkFJkuaWrs9JfA/YKsn29BYJOpTe09eSpDmsa5JIu710APCRqnoxsMvowpIkTYLOSaKtd/1y4GutbEOsRSFJGqOuSeJI4E3AaVV1SZJHAueMLixJ0iTo+jBdf63rB7X9K4DXjjIwSdL4dR3d9PQkl9Kbu4kkT0rysZFGJkkau663mz4I/A/geoCq+jEwdKoOSdLcMJOpwletVXTnLMciSZowXUcorUryDKDaCnWvpd16kiTNXV1bEn9Pb1nR7YHVwK64zKgkzXldRzddR+8ZCUnSJqTr6KaTkswb2N86ySdHF5YkaRJ0vd30xKq6qb9TVTcCTx5NSJKkSdE1SWyWZOv+TpJtmMG0HEk2T3JRkq+2/Z2TnJ/k8iRfaJ3hJLl/21/Rji/s/qNIkmZb1yTxfuA/khyT5BjgP4B/mcHnHME9R0P9M3BsVS0CbgQOa+WHATdW1aOBY1s9SdKYdEoSVXUycCBwDXAtcEDXta2T7AA8Hzi+7QfYCzi1VTkJeFHb3r/t047v3epLksZgJjO5/ozeX/1bACTZqapWdjjvg/RWt3tI2/8T4KaquqPtr6Y3tJb2vgqgqu5IcnOrf90M4pQkzZJOSSLJ4cDb6bUk7gQCFPDEac57AXBtVS1P8px+8RRVq8OxwesuAZYA7LTTTh1+AknS+ujakjgCeExVXT/D6+8B7JdkX2Ar4KH0WhbzkmzRWhM7AFe1+quBHYHVSbYAHgbcsPZFq2opsBRg8eLF90oikqTZ0bXjehVw80wvXlVvqqodqmoh8FLgu1X1cnprURzYqh0CnN62z2j7tOPfrSqTgCSNSdeWxBXAuUm+Bvy+X1hVH1jPzz0K+HySdwEXASe08hOATydZQa8F8dL1vL4kaRZ0TRIr22vL9pqxqjoXOLdtXwE8bYo6twMHrc/1JUmzr+vcTe8ASPKgqvrtaEOSJE0KV6aTJA3lynSSpKFcmU6SNJQr00mShnJlOknSUNO2JJJsDryiPQQnSdqETNuSqKo76c3OKknaxHTtk/h+ko8CXwD+/3MSVXXhSKKSJE2ErkniGe39nQNlRW9dCEnSHNWlT2Iz4LiqOmUDxCNJmiBd+iTuAl6zAWKRJE2YrkNgv53k9Ul2TLJN/zXSyCRJY9e1T+KV7X3w2YgCHjm74UiSJknXWWB3HnUgkqTJ03WN64OnKq+qk2c3HEnSJOl6u2m3ge2tgL2BCwGThCTNYV1vNx0+uJ/kYcCnRxKRJGlidJ4qfC23AYtmMxBJ0uTp2idxJr3RTNBLLLsAPlwnSXNc1z6J9w1s3wH8qqpWjyAeSdIE6ZokVgJXV9XtAEkekGRhVV05ssgkSWPXtU/ii8BdA/t3tjJJ0hzWNUlsUVV/6O+07S2nOynJVkl+mOTHSS5J8o5WvnOS85NcnuQLbUlUkty/7a9oxxfO/EeSJM2WrkliTZL9+jtJ9geu63De74G9qupJ9JY8/cskuwP/DBxbVYuAG4HDWv3DgBur6tHAsa2eJGlMZrLG9ZuTrEyyEjgKeNV0J1XPrW33fu3VX4fi1FZ+EvCitr1/26cd3ztJOsYoSZplXR+m+yWwe5IHA6mqW7p+QFsjeznwaODfgF8CN1XVHa3KamD7tr09sKp95h1Jbgb+hG6tFknSLOvUkkjy7iTzqurWqrolydZJ3tXl3Kq6s6p2BXYAngY8bqpq/Y9ax7HBeJYkWZZk2Zo1a7qEIUlaD11vN+1TVTf1d6rqRmDfmXxQO/9cYHdgXpJ+K2YH4Kq2vRrYEaAdfxhwwxTXWlpVi6tq8fz582cShiRpBromic2T3L+/k+QBwP3XUb9fb36SeQPnPBe4DDgHOLBVOwQ4vW2f0fZpx79bVfdqSUiSNoyuD9P9H+DsJJ+id/vnldzdwbwuC4CTWr/EZsApVfXVJJcCn2+3rC4CTmj1TwA+nWQFvRbES7v/KJKk2da14/pfkvyEXksA4Jiq+maH834CPHmK8ivo9U+sXX47cFCXmCRJo9e1JQG9v/j7Q1gvGk04kqRJ0nV000uAH9LrJ3gJcH6SA9d9liRpY9e1JfG/gN2q6lrodUgD3+HuB+IkSXNQ19FNm/UTRHP9DM6VJG2kurYkvpHkm8Dn2v5fA2eNJiRJ0qToOrrpDUkOAP6C3lPRS6vqtJFGJkkau86jm6rqy8CXRxiLJGnC2K8gSRrKJCFJGmqdSSLJ2e3dxX8kaRM0XZ/EgiTPBvZL8nnWmsq7qi4cWWSSpLGbLkm8DTia3nTeH1jrWH+FOUnSHLXOJFFVpwKnJnlrVR2zgWKSprTynU8Ydwhz3k5v++m4Q9CE6fqcxDFJ9gOe1YrOraqvji4sSdIk6DrB33uAI4BL2+uIViZJmsO6Pkz3fGDXqroLIMlJ9KYLf9OoApMkjd9MnpOYN7D9sNkORJI0ebq2JN4DXJTkHHrDYJ+FrQhJmvO6dlx/Lsm5wG70ksRRVfXrUQYmSRq/mUzwdzVwxghjkSRNGOdukiQNZZKQJA01bZJIslmSizdEMJKkyTJtkmjPRvw4yU4zvXiSHZOck+SyJJckOaKVb5Pk20kub+9bt/Ik+XCSFUl+kuQpM/6JJEmzpuvtpgXAJUnOTnJG/9XhvDuAf6yqxwG7A69Osgu9SQPPrqpFwNltH2AfYFF7LQGOm8HPIkmaZV1HN71jfS7eRkRd3bZvSXIZsD2wP/CcVu0k4FzgqFZ+clUV8IMk85IsaNeRJG1gXZ+TOC/JI4BFVfWdJA8ENp/JByVZCDwZOB/Yrv/FX1VXJ9m2VdseWDVw2upWdo8kkWQJvZYGO+0047tgkqSOuk7w93fAqcD/bkXbA1/p+iFJHgx8CTiyqn6zrqpTlNW9CqqWVtXiqlo8f/78rmFIkmaoa5/Eq4E9gN8AVNXlwLbrPKNJcj96CeIzVfXlVnxNkgXt+ALg2la+Gthx4PQdgKs6xihJmmVdk8Tvq+oP/Z0kWzDFX/hrSxLgBOCyqhpc2e4M4JC2fQhw+kD5wW2U0+7AzfZHSNL4dO24Pi/Jm4EHJHke8A/AmR3O2wN4BfDTJD9qZW8G3guckuQwYCVwUDt2FrAvsAK4DTi0Y3ySpBHomiSOBg4Dfgq8it6X+fHTnVRV/87U/QwAe09Rv+jd2pIkTYCuo5vuagsNnU/vNtPP2xe6JGkO65Qkkjwf+DjwS3otg52TvKqqvj7K4CRJ49X1dtP7gT2ragVAkkcBXwNMEpI0h3Ud3XRtP0E0V3D3sFVJ0hy1zpZEkgPa5iVJzgJOodcncRBwwYhjkySN2XS3m144sH0N8Oy2vQbYeiQRSZImxjqTRFX5nIIkbcK6jm7aGTgcWDh4TlXtN5qwJEmToOvopq/Qm17jTOCu0YUjSZokXZPE7VX14ZFGIkmaOF2TxIeSvB34FvD7fmFVXTiSqCRJE6FrkngCvYn69uLu203V9iVJc1TXJPFi4JGD04VLkua+rk9c/xiYN8pAJEmTp2tLYjvgZ0ku4J59Eg6BlaQ5rGuSePtIo5AkTaSu60mcN+pAJEmTp+sT17dw95rWWwL3A35bVQ8dVWCSpPHr2pJ4yOB+khcBTxtJRJKkidF1dNM9VNVX8BkJSZrzut5uOmBgdzNgMXfffpIkzVFdRzcNritxB3AlsP+sRyNJmihd+yTWa12JJJ8EXkBv+dPHt7JtgC/Qm3b8SuAlVXVjkgAfAvYFbgP+1rmhJGm8plu+9G3rOFxVdcw01z8R+Chw8kDZ0cDZVfXeJEe3/aOAfYBF7fXnwHHtXZI0JtN1XP92ihfAYfS+2Nepqr4H3LBW8f7ASW37JOBFA+UnV88PgHlJFkz7E0iSRma65Uvf399O8hDgCOBQ4PPA+4edN43tqurqdv2rk2zbyrcHVg3UW93Krl7Pz5Ek3UfTDoFNsk2SdwE/oZdUnlJVR1XVtbMcS6Yom3IEVZIlSZYlWbZmzZpZDkOS1LfOJJHkX4ELgFuAJ1TVP1XVjffxM6/p30Zq7/1ksxrYcaDeDsBVU12gqpZW1eKqWjx//vz7GI4kaZjpWhL/CPwp8BbgqiS/aa9bkvxmPT/zDOCQtn0IcPpA+cHp2R24uX9bSpI0HtP1SazXE9l9ST4HPAd4eJLV9GaTfS9wSpLDgJXAQa36WfSGv66gNwR2vYbdSpJmT9eH6dZLVb1syKG9p6hbwKtHGc9T33Dy9JV0nyz/14PHHYKkWXSfWgqSpLnNJCFJGsokIUkayiQhSRrKJCFJGsokIUkayiQhSRrKJCFJGsokIUkayiQhSRrKJCFJGsokIUkayiQhSRrKJCFJGsokIUkayiQhSRrKJCFJGsokIUkayiQhSRrKJCFJGsokIUkayiQhSRrKJCFJGmrikkSSv0zy8yQrkhw97ngkaVM2UUkiyebAvwH7ALsAL0uyy3ijkqRN10QlCeBpwIqquqKq/gB8Hth/zDFJ0iZr0pLE9sCqgf3VrUySNAZbjDuAtWSKsrpXpWQJsKTt3prk5yONarweDlw37iC6yvsOGXcIk2Sj+t0B8Pap/gtusja6319eO6Pf3yO6VJq0JLEa2HFgfwfgqrUrVdVSYOmGCmqckiyrqsXjjkMz5+9u4+bvr2fSbjddACxKsnOSLYGXAmeMOSZJ2mRNVEuiqu5I8hrgm8DmwCer6pIxhyVJm6yJShIAVXUWcNa445ggm8RttTnK393Gzd8fkKp79QtLkgRMXp+EJGmCmCQmlNOTbLySfDLJtUkuHncsmrkkOyY5J8llSS5JcsS4YxonbzdNoDY9yS+A59EbFnwB8LKqunSsgamTJM8CbgVOrqrHjzsezUySBcCCqrowyUOA5cCLNtX/f7YkJpPTk2zEqup7wA3jjkPrp6qurqoL2/YtwGVswjM/mCQmk9OTSBMgyULgycD5441kfEwSk6nT9CSSRifJg4EvAUdW1W/GHc+4mCQmU6fpSSSNRpL70UsQn6mqL487nnEySUwmpyeRxiRJgBOAy6rqA+OOZ9xMEhOoqu4A+tOTXAac4vQkG48knwP+E3hMktVJDht3TJqRPYBXAHsl+VF77TvuoMbFIbCSpKFsSUiShjJJSJKGMklIkoYySUiShjJJSJKGMklIkoYySWjOSDIvyT9sgM95TpJnrFV2ZJKD1+NaRyZ54Hqc984kz52mzguSvGOm15YG+ZyE5ow2GdtXu07P3Z6sTVXdNcPP+Sfg1qp6X9vfArgQeEp7EHIm17oSWFxV101xbPOqunMm11vr/LS49qiq29b3Otq02ZLQXPJe4FHtCdljk5yd5MIkP02yP/QSSVtM5mP0vkB3THJYkl8kOTfJJ5J8tNWdn+RLSS5orz1aIvp74H+2z3kmsBdwYVXdkeRRSS7sB5RkUZLlUwWb5LXAnwLnJDmnld3aWgnnA09P8rb22RcnWdq++ElyYpID2/aVSd4x8LM+FqB6fwGeC7xglv+dtQkxSWguORr4ZVXtCrwBeHFVPQXYE3h//wsWeAy9BYGeDPwReCuwO71Fnh47cL0PAcdW1W7AXwHHV9WVwMdb+a5V9X/pTeOwHKCqfgncnGTXdo1DgROnCraqPkxv4sY9q2rPVvwg4OKq+vOq+nfgo1W1W2sdPYDhX/jXtZ/1OOD1A+XLgGcO/ReTprHFuAOQRiTAu9sqcXfRW49ju3bsV1X1g7b9NOC8qroBIMkXgT9rx54L7HJ3buGhbaWytS2gN8dW3/HAoUleB/x1+4yu7qQ3+2jfnkneCDwQ2Aa4BDhzivP6M5UuBw4YKL+WXmtFWi8mCc1VLwfmA0+tqj+2e/9btWO/Hag31dodfZsBT6+q3w0WDiSNvt8NXBt6X/JvB74LLK+q62cQ9+39fogkWwEfo9dnsar1hWw15Lzft/c7uef/661afNJ68XaT5pJbgP5f+g8Drm0JYk/gEUPO+SHw7CRbtw7ovxo49i16s/ECMHALafBzoNeKeHR/p6pupzeD73HAp2YQ89r6CeG6tgDOgdNcayp/Bly8HudJgElCc0j7i/37SS4GdgUWJ1lGr1XxsyHn/BfwbnrLU34HuBS4uR1+bbvGT5JcSq/DGnq3e1480HH9deBZa136M/RWE/zWNGEvBb7e77heK7abgE8APwW+Qm+dkZnaE/jaepwnAQ6BlUjy4Kq6tbUkTgM+WVWnzfAapwFvrKrL2/7rgYdV1VtnP+LOMW0HfLaq9h5XDNr4mSS0yUvyPnqd1FvR+8v/iJrhf4wkjwG2q6rvtYTxKGCvqZ5/2FCS7Ab8sap+NK4YtPEzSUgbQEscO69VfFRVfXMc8UhdmSQkSUPZcS1JGsokIUkayiQhSRrKJCFJGsokIUka6v8BYOB3VbheGcUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(y_train);\n",
    "pyplot.xlabel('target(y_train)');\n",
    "pyplot.ylabel('Number of occurrences(y_train)');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "随机抽取的数据和原来数据分布类比，各个类型数据分布相似"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prepare cross validation\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "def modelfit(alg, X_train, y_train, cv_folds=None, early_stopping_rounds=10):\n",
    "    xgb_param = alg.get_xgb_params()\n",
    "    xgb_param['num_class'] = 9\n",
    "    \n",
    "    #直接调用xgboost，而非sklarn的wrapper类\n",
    "    xgtrain = xgb.DMatrix(X_train, label = y_train)\n",
    "        \n",
    "    cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=alg.get_params()['n_estimators'], folds =cv_folds,\n",
    "             metrics='mlogloss', early_stopping_rounds=early_stopping_rounds)\n",
    "  \n",
    "    cvresult.to_csv('1_nestimators.csv', index_label = 'n_estimators')\n",
    "    \n",
    "    #最佳参数n_estimators\n",
    "    n_estimators = cvresult.shape[0]\n",
    "    \n",
    "    # 采用交叉验证得到的最佳参数n_estimators，训练模型\n",
    "    alg.set_params(n_estimators = n_estimators)\n",
    "    alg.fit(X_train, y_train, eval_metric='mlogloss')\n",
    "    \n",
    "    print(\"the best n_estimators is: \", n_estimators)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=1000,  #数值大没关系，cv会自动返回合适的n_estimators\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the best n_estimators is:  73\n"
     ]
    }
   ],
   "source": [
    "modelfit(xgb1, X_train, y_train, cv_folds = kfold)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "确定了弱学习器数目为73"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': [6, 7, 8], 'min_child_weight': [4, 5, 6]}"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#max_depth 建议3-10， min_child_weight=1／sqrt(ratio_rare_event) =5.5\n",
    "max_depth = [6,7,8]\n",
    "min_child_weight = [4,5,6]\n",
    "param_test2 = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_test2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.71840, std: 0.03202, params: {'max_depth': 6, 'min_child_weight': 4},\n",
       "  mean: -0.70896, std: 0.04279, params: {'max_depth': 6, 'min_child_weight': 5},\n",
       "  mean: -0.70776, std: 0.03205, params: {'max_depth': 6, 'min_child_weight': 6},\n",
       "  mean: -0.72753, std: 0.04291, params: {'max_depth': 7, 'min_child_weight': 4},\n",
       "  mean: -0.72504, std: 0.03892, params: {'max_depth': 7, 'min_child_weight': 5},\n",
       "  mean: -0.71361, std: 0.04148, params: {'max_depth': 7, 'min_child_weight': 6},\n",
       "  mean: -0.72291, std: 0.04196, params: {'max_depth': 8, 'min_child_weight': 4},\n",
       "  mean: -0.72218, std: 0.03607, params: {'max_depth': 8, 'min_child_weight': 5},\n",
       "  mean: -0.70881, std: 0.04012, params: {'max_depth': 8, 'min_child_weight': 6}],\n",
       " {'max_depth': 6, 'min_child_weight': 6},\n",
       " -0.7077559710847329)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=73,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch2 = GridSearchCV(xgb2, param_grid = param_test2, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2.fit(X_train , y_train)\n",
    "\n",
    "gsearch2.grid_scores_, gsearch2.best_params_,  gsearch2.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "#再次调整弱分类器数目\n",
    "def modelfit(alg, X_train, y_train, useTrainCV=True, cv_folds=None, early_stopping_rounds=100):\n",
    "    \n",
    "    if useTrainCV:\n",
    "        xgb_param = alg.get_xgb_params()\n",
    "        xgb_param['num_class'] = 9\n",
    "        \n",
    "        xgtrain = xgb.DMatrix(X_train, label = y_train)\n",
    "        \n",
    "        cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=alg.get_params()['n_estimators'], folds =cv_folds,\n",
    "                         metrics='mlogloss', early_stopping_rounds=early_stopping_rounds)\n",
    "        \n",
    "        n_estimators = cvresult.shape[0]\n",
    "        alg.set_params(n_estimators = n_estimators)\n",
    "        \n",
    "        print(cvresult)\n",
    "        #result = pd.DataFrame(cvresult)   #cv缺省返回结果为DataFrame\n",
    "        #result.to_csv('my_preds.csv', index_label = 'n_estimators')\n",
    "        cvresult.to_csv('my_preds4_2_3_699.csv', index_label = 'n_estimators')\n",
    "        \n",
    "        # plot\n",
    "        test_means = cvresult['test-mlogloss-mean']\n",
    "        test_stds = cvresult['test-mlogloss-std'] \n",
    "        \n",
    "        train_means = cvresult['train-mlogloss-mean']\n",
    "        train_stds = cvresult['train-mlogloss-std'] \n",
    "\n",
    "        x_axis = range(0, n_estimators)\n",
    "        pyplot.errorbar(x_axis, test_means, yerr=test_stds ,label='Test')\n",
    "        pyplot.errorbar(x_axis, train_means, yerr=train_stds ,label='Train')\n",
    "        pyplot.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "        pyplot.xlabel( 'n_estimators' )\n",
    "        pyplot.ylabel( 'Log Loss' )\n",
    "        pyplot.savefig( 'n_estimators4_2_3_699.png' )\n",
    "    \n",
    "    #Fit the algorithm on the data\n",
    "    alg.fit(X_train, y_train, eval_metric='mlogloss')\n",
    "        \n",
    "    #Predict training set:\n",
    "    train_predprob = alg.predict_proba(X_train)\n",
    "    logloss = log_loss(y_train, train_predprob)\n",
    "\n",
    "        \n",
    "    #Print model report:\n",
    "    print (\"logloss of train :\", logloss)\n",
    "    print(\"the n_estimators is :\", n_estimators)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    train-mlogloss-mean  train-mlogloss-std  test-mlogloss-mean  \\\n",
      "0              1.996123            0.009329            2.001773   \n",
      "1              1.834215            0.007745            1.845504   \n",
      "2              1.703095            0.008919            1.717938   \n",
      "3              1.594197            0.008490            1.613963   \n",
      "4              1.503044            0.006830            1.527470   \n",
      "5              1.420510            0.007191            1.451957   \n",
      "6              1.351173            0.004897            1.388126   \n",
      "7              1.288977            0.006185            1.333048   \n",
      "8              1.232764            0.006702            1.282241   \n",
      "9              1.181158            0.008429            1.232983   \n",
      "10             1.135134            0.008494            1.193044   \n",
      "11             1.093123            0.008727            1.156057   \n",
      "12             1.054856            0.009716            1.120966   \n",
      "13             1.020007            0.009245            1.087982   \n",
      "14             0.988821            0.008849            1.060660   \n",
      "15             0.960039            0.008825            1.036549   \n",
      "16             0.933061            0.008537            1.012330   \n",
      "17             0.907983            0.008112            0.992800   \n",
      "18             0.884234            0.008495            0.972365   \n",
      "19             0.861948            0.009536            0.954377   \n",
      "20             0.841904            0.009787            0.938053   \n",
      "21             0.822647            0.009649            0.923456   \n",
      "22             0.804148            0.009994            0.909895   \n",
      "23             0.787980            0.008707            0.896645   \n",
      "24             0.772328            0.009027            0.884970   \n",
      "25             0.758487            0.009076            0.875239   \n",
      "26             0.745967            0.009114            0.866675   \n",
      "27             0.732232            0.008460            0.857525   \n",
      "28             0.720676            0.008551            0.849483   \n",
      "29             0.709475            0.008274            0.840844   \n",
      "..                  ...                 ...                 ...   \n",
      "43             0.601158            0.007522            0.778394   \n",
      "44             0.595269            0.007241            0.775096   \n",
      "45             0.589934            0.006616            0.772923   \n",
      "46             0.584081            0.006226            0.768678   \n",
      "47             0.578831            0.005759            0.767169   \n",
      "48             0.573805            0.006338            0.765287   \n",
      "49             0.569132            0.006488            0.762597   \n",
      "50             0.564654            0.006104            0.760942   \n",
      "51             0.559997            0.006402            0.760390   \n",
      "52             0.555926            0.006080            0.757464   \n",
      "53             0.551647            0.006450            0.755026   \n",
      "54             0.548147            0.006074            0.753981   \n",
      "55             0.544413            0.006488            0.751918   \n",
      "56             0.541150            0.006861            0.751361   \n",
      "57             0.537865            0.006895            0.750612   \n",
      "58             0.534058            0.006945            0.749417   \n",
      "59             0.530499            0.006955            0.748850   \n",
      "60             0.527169            0.007104            0.748228   \n",
      "61             0.523745            0.007068            0.745563   \n",
      "62             0.520308            0.007284            0.745379   \n",
      "63             0.517193            0.007625            0.744774   \n",
      "64             0.514346            0.007329            0.744205   \n",
      "65             0.511623            0.007581            0.742701   \n",
      "66             0.509179            0.007872            0.742604   \n",
      "67             0.506091            0.007620            0.741298   \n",
      "68             0.503218            0.008095            0.740554   \n",
      "69             0.500486            0.007977            0.740527   \n",
      "70             0.497900            0.008374            0.739512   \n",
      "71             0.495434            0.008526            0.739082   \n",
      "72             0.492748            0.008313            0.737789   \n",
      "\n",
      "    test-mlogloss-std  \n",
      "0            0.010022  \n",
      "1            0.008068  \n",
      "2            0.008750  \n",
      "3            0.010179  \n",
      "4            0.010626  \n",
      "5            0.011163  \n",
      "6            0.012333  \n",
      "7            0.013195  \n",
      "8            0.014876  \n",
      "9            0.013055  \n",
      "10           0.013885  \n",
      "11           0.014911  \n",
      "12           0.014172  \n",
      "13           0.014515  \n",
      "14           0.014992  \n",
      "15           0.015077  \n",
      "16           0.014444  \n",
      "17           0.015789  \n",
      "18           0.016954  \n",
      "19           0.017513  \n",
      "20           0.018500  \n",
      "21           0.020636  \n",
      "22           0.022536  \n",
      "23           0.023228  \n",
      "24           0.023642  \n",
      "25           0.023681  \n",
      "26           0.024057  \n",
      "27           0.025432  \n",
      "28           0.025574  \n",
      "29           0.025772  \n",
      "..                ...  \n",
      "43           0.032638  \n",
      "44           0.033921  \n",
      "45           0.035930  \n",
      "46           0.034734  \n",
      "47           0.036240  \n",
      "48           0.035977  \n",
      "49           0.036550  \n",
      "50           0.034591  \n",
      "51           0.035027  \n",
      "52           0.035049  \n",
      "53           0.034247  \n",
      "54           0.035083  \n",
      "55           0.035467  \n",
      "56           0.035800  \n",
      "57           0.036308  \n",
      "58           0.036251  \n",
      "59           0.038314  \n",
      "60           0.038748  \n",
      "61           0.038167  \n",
      "62           0.038562  \n",
      "63           0.037857  \n",
      "64           0.038127  \n",
      "65           0.039969  \n",
      "66           0.039722  \n",
      "67           0.039808  \n",
      "68           0.041309  \n",
      "69           0.041998  \n",
      "70           0.042396  \n",
      "71           0.043055  \n",
      "72           0.043289  \n",
      "\n",
      "[73 rows x 4 columns]\n",
      "logloss of train : 0.461440149809799\n",
      "the n_estimators is : 73\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEXCAYAAABCjVgAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd8VfX9+PHX+47sQICEDQZkiCIgIuCmiIpa98RR96q2amsdrT9r7dKvrW2tilIHtg7cintPlgRkb5lhJYwQssd9//74nMRLSCCB3Nyb5P18PM4j94x7zjv3Jvd9P+N8PqKqGGOMMQC+aAdgjDEmdlhSMMYYU82SgjHGmGqWFIwxxlSzpGCMMaaaJQVjjDHVLCkYE0ZEfisiT0U7DmOixZJCMyMiKSKyWkQuDtuWKiJrReS8sG3DRORdEdkuInkiskhE/iwi7bz9V4hIpYgUeMtKEbkxwrGPEpHsSF6jIWqLR1X/oqrXROh6q0VkTCTOHQlN9X41t9elpbOk0MyoagFwHfAvEcnwNv8fkKWqrwGIyFHAl8AU4CBVTQPGAhXA4LDTTVPVFFVNAc4D/k9EDmua38Q0hIgEoh2DaSVU1ZZmuAATgZeAUcBWoEvYvm+Bf+/l+VcA39bY9h1wcdj6GcBCIA+XZAaE7RvgbcvzjjkjbN+pwCJgJ7AeuB1IBoqBEFDgLV3r+L0eA97znj8DOLAer8dBwCfANmApcMG+xAPcBzzvPS8TUOBKYB2wHbgBOAKY5/3uj4Zd50Dgc+/92AK8AKR5+/7nXavYu9Yd9XiNVwN3etcqBQLe+nrvd1kKnFDLazES2AT4w7adDczzHg8HsoB8YDPwcB2v6Sggu459bYH/ArnAGuAewOft8wN/916DVcDN3usYqONcq4Exdey7Fljhva+Tq/5mAAH+AeQAO7zXaGBd73e0/1+b0xL1AGzZxzcO2gEbvX+8K8O2JwOVwKi9PP8KwpKC90GXB/Tz1vsBhcCJQBC4w/vnjPPWVwC/9dZHe/+A/b3nbgSODYtzqPe4zg+ZsDgmeh8Aw70PwReASXt5TjLuQ/tK7zlDvdflkIbGQ+1J4QkgATgJKAHeAjoC3bwPpeO94/t4r1c8kAF8Dfwz7Ny7fPjt6TUOO34O0ANIBPp7v2fXsPhqTZjAD8CJYeuvAnd5j6cBl3mPU4CRdZyjzvcLlxDeBlK9OJYBV3v7bsB9KHf3Xu9P2Yek4P1dbfHez3jg38DX3r6TgVlAGi5BDMD7YlTX+21L/RarPmqmVHU77htmEvBG2K52uGrBTVUbROT/vHaFQhG5J+zYkd72Alwp4X/Acm/fhcB7qvqJqpYDf8N9MB2F+yaaAjygqmWq+jnwLjDOe245cLCItFHV7ao6u4G/3huq+p2qVuCSwpC9HP9TYLWqPquqFd71XsdViTVGPH9U1RJV/Rj3If6Squao6nrgG+AwAFVd4b1epaqaCzwMHL+H8+7pNa7yiKquU9ViXLKP936XoKquVtUf6jj3S3jvh4ik4r49vxT2evQRkXRVLVDV6Q15MUTE78V+t6ruVNXVuJLBZd4hFwD/UtVs7+/0gYacP8wlwDOqOltVS4G7gSNFJNP7HVJxJURR1cWqujHs99uf97tVs6TQTInIpbhvaJ8CD4bt2o6rpuhStUFV71DXrvAm7pt0lemqmqauTaEzcAjwF29fV1y1QNU5Qrhvqd28feu8bVXWePsAzsV9CK0Rka9E5MgG/nqbwh4X4RLQnhwAjPASXJ6I5OE+UDo3Ujybwx4X17KeAiAiHUVkkoisF5F84HkgfQ/n3dNrXGVd2P4VwK240kyOd62udZz7ReAcEYkHzgFmq2rVta7GlVKWiMhMEfnpHmKsTTquhLgmbFv4+981PO4ajxui5utTgKua6+Z9EXkUV9W4WUQmiEgb79D9fb9bNUsKzZCIdMTVp14LXA9cICLHAahqIa4e/pyGnFNVN+O+XZ/ubdqA+7CtuqbgqjHWe/t6iEj4309Pbx+qOlNVz8RVsbwFvFJ1mYbE1ADrgK+8BFe1pKjqjU0cz1+9cw5S1TbApbiqjSo1r7en17jW56jqi6p6jPc8ZdcvBOHHLcJ9oJ4CXIxLElX7lqvqONzr8SDwmogk1//XZAvu2/gBYduq339c9U33sH09GnDucDVfn2SgAz/+nT2iqofjvsz0A37jba/r/Tb1YEmheXoUeEtVv/CKzHcA//G+FeKtXyUid3kJBBHpDvSq64Qi0gHXGLnQ2/QKcJqInCAiQeDXuMbOqbikUwjcISJBERmFSyaTRCRORC4RkbZelUg+rtoD3DfsDiLStpFehyrvAv1E5DIvnqCIHCEiA5o4nlRcI3KeiHTD+5AKsxnoHba+p9d4NyLSX0RGe+9zCa6UUlnbsZ4XgV8Cx+HaFKrOc6mIZHglkzxvc53nEZGE8AVXEn0F+LPXHfoA4Fe4klHV73WLiHQTkTRc4/jeBGtcJ+DFf6WIDPF+578AM1R1tff+jvBet0Lv9ajcy/tt6iPajRq2NGwBzsJ9g0qrsf0z4M9h6yOA93H/9HnAAuDPQAdv/xW4f5aqnjc5uDrnjmHnOBvXYLgD+Aqv4dbbd4i3bYd3zNne9jjgQ1w1Vj4wEzgm7HnP4KoA8qi799GfwtZHsZfGae+4/rgeS7ne+T/HtUU0KB5qb2gOhB2fTVgjPu6D8J6w12SW93rOwX3IZ4cdeyaw1rvW7fV4jVeza8P0IFzbz05cY/y7tb2GYcf3xH2Av1dj+/Pe+12A+xJwVh3PH+X9/jWXPri2q+e913sdcC8/9j4K4EqyW3G9j27DlSykjuusruUaf/L23YBrNK/6fbt720/A9Tgq4MeeXil7e79t2fsi3gtsjDERISKnAE+o6gF7PdhEnVUfGWMalYgkisipIhLwqtF+j+vkYJoBKymYZkFEjgU+qG2fut5TJkaISBKuKuwgXLvHe8Atqpof1cBMvVhSMMYYU82qj4wxxlRrdoNspaena2ZmZrTDMMaYZmXWrFlbVDVjb8c1u6SQmZlJVlZWtMMwxphmRUTW7P0oqz4yxhgTxpKCMcaYapYUjDHGVLOkYIwxppolBWOMMdUsKRhjjKlmScEYY0y1iCUFEekhIl+IyGIRWSgit9RyjIjIIyKyQkTmicjQSMVjjDFm7yJZUqgAfq2qA3Bz+t4kIgfXOOYUoK+3XAeMj1QwFzwxlTP+/S2hkI31ZIwxdYlYUlDVjepNmK2qO4HF7Dr3LLhJR/6rznQgTUS6EAFjN41nyKZXWLVpayROb4wxLUKTtCmISCZwGG4ax3Dd2HVS72x2TxyIyHUikiUiWbm5ufsUw2DfSu4PPseaxTP36fnGGNMaRDwpiEgKbkL4W2sZT11qecpu9TuqOkFVh6nqsIyMvY7nVKshnRMBKFpl4yYZY0xdIpoUvEm1XwdeUNU3ajkkG+gRtt4dN/9wo/Nf+wn5vrbEb5kXidMbY0yLEMneRwI8DSxW1YfrOGwy8DOvF9JIYIeqboxQQOSmDqB78VLKKkIRuYQxxjR3kRw6+2jgMmC+iMzxtv0W6Amgqk8A7wOnAiuAIuDKCMZDqPMQ+uZ9x5LsXAZmdorkpYwxplmKWFJQ1W+pvc0g/BgFbopUDDW16zOcwNInWLdkBgMzz2iqyxpjTLPRqu5o7tB3OAAlq2dFORJjjIlNrSopSNvu7PClkbR1frRDMcaYmNSqkgIibGt7CAeULqOgtCLa0RhjTMxpXUkBkK5D6CvZLFqzKdqhGGNMzGl1SaF93xH4Rdm49Ltoh2KMMTGn1SWFNr1dY3PZWmtsNsaYmlpdUqBNF3b425OybUG0IzHGmJjT+pICkJd2CAeWL2drQWm0QzHGmJjSKpOCv/tQ+sgGFq6KyDBLxhjTbLXKpJDedwQ+UTYus2G0jTEmXKtMCgkHHA5AZfbsKEdijDGxpVUmBVI7kxfIIC1vAW74JWOMMdBakwKws/0h9Kv8geztxdEOxRhjYkarTQpxPQ6nt2zkpme+jHYoxhgTM1ptUujQdzg+UX6y481oh2KMMTGj1SaFQI9hAKSI3atgjDFVIjkd5zMikiMitd46LCJtReQdEZkrIgtFJKKzru0mOZ3tSb3oo6vJyS9p0ksbY0ysimRJYSIwdg/7bwIWqepgYBTwdxGJi2A8uwkdcAxH+JYyfcXmprysMcbErIglBVX9Gti2p0OAVBERIMU7tkknOWh38GhSpIS1C6c15WWNMSZmRbNN4VFgALABmA/coqqhpgzAl3k0AP61U5ryssYYE7OimRROBuYAXYEhwKMi0qa2A0XkOhHJEpGs3NzcxosgtRN5yb3oXzKP7O1FjXdeY4xppqKZFK4E3lBnBbAKOKi2A1V1gqoOU9VhGRkZjRqEHnC0a1dYbu0KxhgTzaSwFjgBQEQ6Af2BlU0dRNsBo0mVYtYtnt7UlzbGmJgTiNSJReQlXK+idBHJBn4PBAFU9Qngj8BEEZkPCHCnqm6JVDx18WUeA0Bw7VRUL8K1extjTOsUsaSgquP2sn8DcFKkrl9vqZ3YkZzJwfnzWLO1iMz05GhHZIwxUdNq72gOJ5nHMMy3lGnWrmCMaeUsKQCpB/2ENlLMusUzoh2KMcZElSUFXEkBIC57is2vYIxp1SwpAKR2Jj85k0PLF7A8pyDa0RhjTNRYUvD4eh3DEb4lXP+cVSEZY1ovSwqelP6uXaFz0fJoh2KMMVFjSaHKAW4cpEHlcyksbdJx+YwxJmZYUqjSpgvF/lSO9i3gq2WNOL6SMcY0I5YUwsQf8TNG+hfz1bwmH23DGGNigiWFML7+pxBHBaXLP6O8sklH8TbGmJhgSSFcz5GUB1M5umIm01dujXY0xhjT5CwphPMH8fU9kdH+OXyyYGO0ozHGmCZnSaEG/0Gn0kHyWb/wW0Ihu7vZGNO6WFKoqe8YQuLnsJLpzFu/I9rRGGNMk7KkUFNiO0LdRzDGN5uPF26KdjTGGNOkLCnUInDQKRzkW8f38+dFOxRjjGlSlhRq0/8UAPrkTWGFDZBnjGlFIpYUROQZEckRkQV7OGaUiMwRkYUi8lWkYmmw9L5UpPV2VUiLrArJGNN6RLKkMBEYW9dOEUkDHgfOUNVDgPMjGEuDBQacypG+RTz7+fxoh2KMMU0mYklBVb8Gtu3hkIuBN1R1rXd8TqRi2Sf9xhInFQytmMuKnJ3RjsYYY5pENNsU+gHtRORLEZklIj+LYiy76zmSkAQY45vFa7PWRzsaY4xpEtFMCgHgcOA04GTg/4lIv9oOFJHrRCRLRLJyc5toBFN/EF9iO8YGsnh71moqbCwkY0wrEM2kkA18qKqFqroF+BoYXNuBqjpBVYep6rCMjIymi/DMf5NKEQOKZvLN8i1Nd11jjImSaCaFt4FjRSQgIknACGBxFOPZ3YEnoIntuSB+Gq/OWhftaIwxJuIi2SX1JWAa0F9EskXkahG5QURuAFDVxcCHwDzgO+ApVa2z+2pUBOKQgedwAjOZtmg12wvLoh2RMcZEVCBSJ1bVcfU45iHgoUjF0CgGXUhw5lOM1u+YPPdwLj8qM9oRGWNMxNgdzXvT/Qhol8klSTOsCskY0+JZUtgbETj0AoZUzCVn/RoWb8yPdkTGGBMxlhTqY9AF+AhxVnAar83KjnY0xhgTMZYU6iO9L3Q9jLP9U3hu6mrKKuyeBWNMy2RJob4GXcgAVpGp6/jApuo0xrRQlhTqa+C5qPi5IuU7npmyOtrRGGNMRFhSqK+Ujkh8G86q+JB567Yxe+32aEdkjDGNzpJCQ6RkkKIFjI5fxkQrLRhjWiBLCg1x/TeQkMav2k/l/fkb2bSjJNoRGWNMo7Kk0BDBBBh8EQfv+Jo2uoPnp6+JdkTGGNOoLCk01NDLkVAZd3aZw4vfraWkvDLaERljTKOxpNBQnQ6G7sM5veJjthWWMnnuhmhHZIwxjcaSwr44/HKS8ldybod13Dd5IRc8MTXaERljTKOwpLAvDjkb4ttwc9oUisoqyS+piHZExhjTKCwp7Iu4ZDj0fDI3f0IHfyHrtxejqtGOyhhj9pslhX11+BVIRQnjD13BztIKvl1h03UaY5o/Swr7qssg6HoYw7a9Q9c28fz942VWWjDGNHuRnI7zGRHJEZE9TrEpIkeISKWInBepWCJm6OX4chbxh+BzzFmXxxdLc6IdkTHG7JdIlhQmAmP3dICI+IEHgY8iGEfkHHo++AKcEJpCz/ZJPPyJlRaMMc1bxJKCqn4NbNvLYb8AXgea51fs+BQ49tf4irfyu+HCgvX5fLRwc7SjMsaYfbbXpCAiB4pIvPd4lIj8UkTS9vfCItINOBt4Yn/PFVXDr4dgEidum0Tv9GRue3mO3bdgjGm26lNSeB2oFJE+wNNAL+DFRrj2P4E7VXWv40SIyHUikiUiWbm5uY1w6UaU3MG1LSx4lbuOSqa4vJJthWXRjsoYY/ZJfZJCSFUrcN/q/6mqtwFdGuHaw4BJIrIaOA94XETOqu1AVZ2gqsNUdVhGRkYjXLqRHXkTACfmvUpSnJ+124opLrMxkYwxzU99kkK5iIwDLgfe9bYF9/fCqtpLVTNVNRN4Dfi5qr61v+eNirQeMOhCZPZ/GdSunLLKEOO/XBHtqIwxpsHqkxSuBI4E/qyqq0SkF/D83p4kIi8B04D+IpItIleLyA0icsP+hRyjjr4FKkqYNHguZw3pyhNfr2TN1sJoR2WMMQ0iDelCKSLtgB6qOi9yIe3ZsGHDNCsrK1qX37NJl8Dqb8i5ZjY/eSSLIw/swFOXHxHtqIwxBhGZparD9nZcfXoffSkibUSkPTAXeFZEHm6MIFucY34FJTvo+PwJ3DKmL58uzuHzJdZF1RjTfNSn+qitquYD5wDPqurhwJjIhtVMdT8cDjwBSndwxeEdODAjmRufn835462LqjGmeahPUgiISBfgAn5saDZ1OeH/QfF24r4bz31nHEJpRYiN+TaXszGmeahPUrgfNwzFD6o6U0R6A8sjG1Yz1vUwGHAGTHuUY7sK7ZKCrM8rZtUWa3Q2xsS+vSYFVX1VVQep6o3e+kpVPTfyoTVjo++B8iL49h9kdkjGJ8Ldb8yzcZGMMTGvPg3N3UXkTW/E080i8rqIdG+K4JqtjP4weBx89x/evDSTP501kOkrt/HyzHXRjswYY/aoPtVHzwKTga5AN+Adb5vZk1F3gYbgqwe56IgejOzdnj+/v5gca18wxsSw+iSFDFV9VlUrvGUiEINjTcSYtJ4w7Cr4/nlk20r+es4gyipC3Pv2wmhHZowxdapPUtgiIpeKiN9bLgW2RjqwFuG42wGFp0+iV3oyt47px4cLN3Hiw19FOzJjjKlVfZLCVbjuqJuAjbjB666MZFAtRkpHaNMNirbAmmlcc2wvkuL8rN5ayI6i8mhHZ4wxu6lP76O1qnqGqmaoakdVPQt3I5upj5tmuMTwwR0ERemdnkx5pXL/u4uiHZkxxuxmX2de+1WjRtGSxSXDiffDpnnw/f9Ijg/QLS2B12dn2xAYxpiYs69JQRo1ipZu4LnQ80j47H5e/tkAvrj9J/TvlMrdb8xnR7FVIxljYse+JgW7C6shROCUB6FoG3z1IHEBHw+dP4gtBWX8+T2rRjLGxI46k4KI7BSR/FqWnbh7FkxDdBkMQ38G302A3KUM6p7GDcf35pWsbE7+h/VGMsbEhjqTgqqmqmqbWpZUVQ00ZZAtxgn3grouqqjyyxP6khj0s2pLkfVGMsbEhH2tPjL7Ijnd3dRWkgcL3yA+4Kd3RjLllSF+Mel7KkNWK2eMia6IJQURecYbL2lBHfsvEZF53jJVRAZHKpaYcnMWdBkCH9wJxdtJiQ+QmZ7M18tyeeCDxdGOzhjTykWypDARGLuH/auA41V1EPBHYEIEY4kd/gCc8YhrdP7kXl6+/ki+uH0UVxyVyX++WcXrs7KjHaExphWLWFJQ1a+BbXvYP1VVt3ur04HWM/Jql8Fw5M9h9n9h9RQA7jltAEf36cDdb8xn9trtezmBMcZERn2Gzq6tF9I6bzjt3o0Ux9XAB3uI4ToRyRKRrNzc3Ea6ZJSNuhvSDoB3boGKUgJ+H4+OG0rntglc+OQ0zn5sSrQjNMa0QvUpKTwM/AY3bHZ34HbgP8Ak4Jn9DUBEfoJLCnfWdYyqTlDVYao6LCOjhQzQGpcMP30Yti6Hbx4GoF1yHE9dPozKkLJ08052lliPJGNM06pPUhirqk+q6k5VzVfVCcCpqvoy0G5/Li4ig4CngDNVtfWNvNpnDCRnwFcPwmY3pHa/Tqn07ZhKcVklNz4/m7KKUJSDNMa0JvVJCiERuUBEfN5yQdi+fe5DKSI9gTeAy1R12b6ep9lr1xt8AXj9Gih3E/B8dNtxPHT+YL5dsYW7bBpPY0wTqs9NaJcA/wIe99anAZeKSCJwc11PEpGXgFFAuohkA78HggCq+gRwL9ABeFxEACpUddi+/RrN2DUfw/JP4IXz4LM/wNi/AnDe4d3ZmFfM3z9ZxvSVW+nRLomXrz8yysEaY1q6vSYFVV0JnF7H7m/38LxxeznvNcA1e7t+q9D3RBh+PUx/HPqc4KqVgJtH92HDjmJe+m4dcX67z9AYE3n16X3U3etplCMim0XkdRFpPd1Hm8qJf4CMAfDWz6FwCwAiwh/PHEhaYpDVW4t4d96GKAdpjGnp6vP181lgMm4QvG7AO94205iCiXDuU1C8HSb/wo2RBAT8Pvp0TCE1PsBtL8/hy6U5UQ7UGNOS1ScpZKjqs6pa4S0TgRbSLzTGdB4IY+6Dpe/Dvw+v3vzajUcx5e7R9OuUyg3PzyJrdZ33BBpjzH6pT1LYIiKXiojfWy4FWl/30aYy4kZIbAfbVkJ2VvXmNglBnrtqOF3bJnLhhOn89JFvohikMaalqk9SuAq4ANgEbATOA66MZFCtms8Hv5gNaT3glZ9Vty8ApKfE879rRuAXYcmmnSzfvDOKgRpjWqK9JgVVXauqZ6hqhqp2VNWzgHOaILbWK6k9XPA/lxBeuwpCldW7uqUlclCXVERg3H9msCLHEoMxpvHsaz/HXzVqFGZ3XYfAaX+HVV/B53/aZdfkm4/hg1uOA+CiCTNYkVMQjQiNMS3QviYFadQoTO2GXuam8Pz2YXhs5C67+nRMYdJ1IwDllH99zRmP1nnLiDHG1Nu+JgUbd6GpnPIQxKXAlmWwaf4uu/p0TOXFa0eiCos35rNww44oBWmMaSnqTAp1DJmdLyI7cfcsmKYQTHCztaV0ghcvhPyNu+zu1ymVAV1SEYTzxk/jo4WbohSoMaYlqDMpqGqqqrapZUlV1fqMmWQaS5sucPHLUJwHL10IZYW77E6KC3BI1zb06+zuYxj/5Q9c8MRULnxyWpQCNsY0V/bh3lx0GQTnPQOTxsHr18KF/wOfH6B6oLyS8kpuf3UuD364hPSUOHp1SI5mxMaYZshGWWtO+o+Fk/8KS9+DfwzcbXdC0M+/xx3GbWP6saWgjIUb81mZaz2TjDH1Z0mhuRlxPaR2gZ0bqmdsCyci3DKmL/06pVBWEeL0f3/Lm99nRyFQY0xzJM1tApdhw4ZpVlbW3g9syUIhePM6mP8qnPYwHHF1rYdt3FHMLS/N4bvV20hPiSOzQzKv3XhUEwdrjIkFIjKrPnPWWEmhOfL54Kzx0G8svPdrmPdqrYd1aZvIi9eO4Jej+7CloIwFG3awZFN+EwdrjGlOLCk0V/4gnD8RMo+BN67Z7ea2KgG/j1+d1J+DOqdSGVLOfHQKL3231qb4NMbUKmJJQUSe8SbmWVDHfhGRR0RkhYjME5GhkYqlxQomwriX3M1tuUtg2cd1Hto2Mcih3doyvFd77n5jPof/6VPOHT+1CYM1xjQHkSwpTATG7mH/KUBfb7kOGB/BWFqu+FToeAjEJcOki2HR27Ue9vL1R/LGz4/muSuHc8fY/mwrLGN+9g4+XLDRSg3GmGoRbWgWkUzgXVXdrf+kiDwJfKmqL3nrS4FRqrqx5rHhrKG5DiU74IXzIXuma28YfNEeDz/1X1+zemsRRWWVHNs3nR3F5SQG/dX3PBhjWpbm0NDcDVgXtp7tbduNiFwnIlkikpWbm9skwTU7CW3hsjch81h48wbIemaPh6cmBBnYtQ33nX4wc9bmMT97B2u3FVFcVrnH5xljWrZoJoXaRlqttdiiqhNUdZiqDsvIsJlA6xSXDBe/An1Pgndvg38Oqp7ruaaXrz+SV244iiuO7sXnt4+iQ0ocG3eUcNI/v7J5oI1pxaI5zEU20CNsvTuwIUqxtBzBBLjwefjHwZC3Bt77lRtp1V/3W52RGs+BGSlkpJRTVhniimdn8tNBXVi/vZi4gM+qlIxpRaKZFCYDN4vIJGAEsGNv7QmmngJx8Otl8Pkf3VwM+RvcuElxdY+FVPXBX1pRyZNfreTRL1ZQGVK6pSVQUl5JQtDfVNEbY6IoYg3NIvISMApIBzYDvweCAKr6hIgI8Ciuh1IRcKWq7rUF2RqaG2jmU/D+byCYBB0Phms+qdfTVuYWcPbjU9lRXE73don85uT+vDB9DSJiJQdjmqH6NjRHrKSgquP2sl+BmyJ1feM54hpI7QovXwIb57qJejofuten9c5I4aDOqewoLscnwi2T5pAc56dH+yRUFZfTjTEtjY191Fo8cTzkLgJfEM55EgacXu+nhkLKm9+v5+435lNWGWLYAe24dUw/HvlsmZUcjGkmmkOXVNOUbvgKbp0PHQ+Cly+Frx6qs2dSTT6fcO7h3Rncoy2ZHZJYn1fMpU/PYNHGneQVldnNb8a0IFZSaG3KS+CdX8K8lyEpHW6Z4+6KboDSikpezcrm/ncWUVYZom/HFK45thevZWXj81nJwZhYZCUFU7tgApz9JKRlQtEW+M9oyFncoFPEB/xcOvIABvdoS+/0ZIJ+H3e+Pp/v1+WRvb2YnPwSAC58cppNCWpMM2NJoTUSgVvnwuXvQPF2lxjqGH57T3wiZKTG894vj+HFa0eQEh9gfV4xRz3wOTe9OJv84nKrWjKmmbHqo9YufyO8dhWsnQopneGWua40sQ8ufHIaJeWVDO/VnleysqvHU7pzbH/OHtq5SxTOAAAbyElEQVSdtonB6pKDVTEZ07Ss+sjUT5sursTQphsUbPKqk5bs8+kSgn5+d9rBzPjtCfROT8YncN87ixjxl0+547W5FJRW7FJ6sComY2JLNO9oNrHCH4BfLYLln7jB9CaMglMegKGXu6qmegr/9p8Q9JORGk9Gajz3nHYwL363hrfnbKCorJLEoJ/HvljBmUO67nYOK0kYE11WUjA/6nsi3DgVeo6Ed26Bhw6Ep07a79Me2r0tfz1nEDN+ewKZHZII+ISHPlrKMQ9+waIN+WzaUcK6bUW1PtdKEsY0LSspmF2ldoJL34Bp/4ZPfg+l37sSRN8TG3yqmt/2UxOCdGqTQKc2Cfzt/MG8PWc9j33xA2u2FXHs/31Bv04p7CypIC0xSFlFiLhA7d9ZrDRhTORYScHszueDo2+B67+CDn3ghfNg8i+hdGejXaJH+yRuHt2XQd3bMrh7W/7fTw8mIzWeTTtKWLxpJ4P/8DGXP/MdG/KKKSytIBSqu0OElSaMaTxWUjB16zIYrvsSvvgLTH0E5k6CDn3h51P2+ZS1fbtPCPq5+pheXH1ML84dP4X84gqOOrADU37YyrrtxazbXszwv3zGcX3TOb5/BuWVIYJ++z5jTCRYUjB7FoiHE/8A/U+F/54BOQvgtavh5L+4qqb9VDNJBHw+2ifH8Ycz3QyuZz82hR3F5Qzs1pYvlubwxvfrAUiK83Pf5IWM6NV+tyRRs3rJqpuMqT9LCqZ+eo6AO9fAt/9wczQs/9gNk5HaBa56v9EuU/ODOy7gIyM1nkfGHUZlSFmwfgc3vTCb/JJyXp65jolTVwOQGPRz/zuLOKZvBypDit9Xd68pSxLG1M2Sgqm/YAL85G449Hx4/9ew8kso3AxrZ7ikEWF+nzC4Rxrd2iXSjUT+d/UI5q/fwS2Tvie/uJwXZqzhmSmrEFxJ4p635nNot7YUllaQGLfnSYKsdGGMY0nBNFx6H7jsLXhsBGxfBc+cBIMuctVMqZ0b9VJ7+lCOC/g4/IB2dEtLpFtaIs9dNZzZa7Zz+6tz2Vlawdvfb+D56WsBd7vFOY9PYUiPdmwtKCU5PkAopPj2UKIIZ0nCtBaWFMy+EYGbv4PSAvjm7zDtUZj/KrTt4bYH4iN26bo+mBOCfo7qk06P9kkAvHTtSNZsK+La52ZSUFqJT4QXZqyhtCIEwMD7PqJPxxT6dExhQ14xSXF+thWW0T45bq8x1JYkrLRhWoKIJgURGQv8C/ADT6nqAzX29wSeA9K8Y+5S1caroDaRF58CY34Ph10KT50AeatdCeKkP8JBP23QHdH7qq4PXZ9P6JWeTIeUeDqkuOPKK0Oc/dgUCksrGHVQR1bkFDB1xVY2eSO7Dv3jJ3RLS6SorIKkOD/vzdtI304phFTx7efvYknDNAcRSwoi4gceA04EsoGZIjJZVReFHXYP8IqqjheRg4H3gcxIxWQiqMOBcOdqWPEpfPQ7N5FPfBto19tN8NOE9vQhG/T7SI4PkBwf4PenH1K9/dzHp1BYVsk5Q7sxf30+nyzcxPaicm56cXb1MQlBHzf8bxb9OqWwtaCUhKCfnJ0ltE+KI9BIXWT3ljgskZhIi2RJYTiwQlVXAojIJOBMIDwpKNDGe9wW2BDBeExT6DMGeo2C2RPhgzth0xx4+TI4/k7oPDAqIdXnAzTg99E20cd1xx0IuA/fUEi578xDWJFTwIMfLKG4vJJlm3fy8aJNVN1LN/zPnyECaYlBSsrdXdi/fXM+3dIS2VJQSnzAR05+CRmpkalOq0/SsERiGiKSSaEbsC5sPRuo2UXlPuBjEfkFkAyMqe1EInIdcB1Az549Gz1Q08j8ATjiGjdHQ/5610tp8WRXnXT8ndBlUFTDq++Ho88nHNK1LYd0bcuLM9ZWP7ekvJLzn5hKSXmInx15AFsKythaWMp78zZSVhHiwwWb2FZYVn2e4X/5jPiADxGI8/v49Stz6dw2nk35JcT5fcxas430lHg6pMSjqkiEq9z2pTRiiaX1iGRSqO0vu+ZYBeOAiar6dxE5EvifiAxU1dAuT1KdAEwAN59CRKI1je/qj9zP4u0w/QmYPh6WvAuJ7eDiV6HHEdGNL0xDPuwSgn6S4gIkxcFlR2ZWb1++uaD6XIWlFVw0YRplFSEuHXkA2duLeTVrHaWVIab+sIWcnaVUesWNc8f/OESHCAR8wui/f0lqQpDVWwoI+n08/MkyuqUlsKO4nDi/j/ySclLjo9dPxKq5Wq5I/lVlAz3C1ruze/XQ1cBYAFWdJiIJQDqQE8G4TFNLbOfubxh5IzxxLOzcAE+Pgcxj4bjbodfxTdIg3RD7+2GWHB/YLXHMWZdXfe7KkHLe+KmUV4a4/eT+rrRRUMrEqaupqFQGdG5Dfkk5FSGluKyCRz9fTvjwT4Pu+5iEoA9VV/q46cXZdEpNYENeMUG/j8+XbKZtYpC2iUHKK0ME6tn1NlL2pbfWvpRgonGOfUmA9XlOtBJrJJPCTKCviPQC1gMXARfXOGYtcAIwUUQGAAlAbgRjMtGUmAa3zXfdWGdNhKn/hv+eCXHJcMpDMPDcfZ71LdIa+x/T7xPiAj7iAj5G9e9Yvf3zJe770GOXDAV+/GB4/poRbM4v4drnsiirDDFueE8255fwxuz1lFWGWLwhny/ycygqqwTgqom7zk4oAqP//iU92iWxakshQb8w4esfSIwLkLuzFL9PmLpiC6kJQUrKK/H7pEH3cZhdNTRxxFKSiFhSUNUKEbkZ+AjX3fQZVV0oIvcDWao6Gfg18B8RuQ1XtXSFNrf5QU3DxafAUTe7dofxR7uSw9s/h4/vgWCSuwHu2s+iHeUe1faPWXNbY/7zBv0+urdLok1iEIBrju0NwLzsHbtc67zxUymrDHHfGYewo7ic/OJyHv54GWWVIfp3SmXd9iK2FpZRGVL+8v6uM+xd/NSMXdb7/O592iXF0T45jk35Jfh9wi2Tvic5PsDabUUI8M9PlxHwCRvyivH5hPfnbyQ9JZ7i8kqCPmmSNhLTuCJaKendc/B+jW33hj1eBBwdyRhMDAsmwC9ngSqs/gZmPOnaHPKz4X9nu3sf+p8Ws6WHhopk0qji9wmJPj9De7ar3lbVSD7+0sMB941TVXnmyuEUlVVw7XNZVIaU3512MDtLynnggyVUhpQzhnRla2EZ2wrK2JRfQllFiDnr8igsrWBbYRkhhX9+unyX6//8hdm7rPf53Qe0SwpSUFpBwOfj0qdmkBD0ER/w80NuAX4R/vbRUtokBsjZWYJPhHfnbSDg87G9qAwBpq7YQjDgo6C0AgF+yC0gMegnIeinMqSIYMmnEdkdzSb6RKDXcW75zwlQkANbVsBrV4EvAMkZbliNjgdFO9KIa4rEASAipMQHSIkPkBB040IdeWAHAJ7+dhUAvz6pf/XxdVWHvHTtSCpVGTdhOpUh5c9nH8qWglL+8M5CyitDnD64K9uLyvlw/kYqQkpRWQXbi0KUVoTYWVJBZUh5/MsVu7SX3Pzi97vEWrMEc8Lfd7/vpdfd7+MTV90gwKD7PiLoNcj7fcK546fSPjmOlbmFBPzCAx8sQQR8Auu2FyEIj36+nIDfx8YdJYjA89PXEPQLuTtL8Ql8vmQzSXEBCksrEIHlm3ciAsVllSCwIa+YxKCfxDh/s05SlhRMbKmqNgqFYNVX8Pq1sHMTPD7CJY0jrnXDePtb559uUyWN+vL5BB+C3+eWg7u6247SU9x9Gb852SXyH3J+7JlVpSqxTLpuJIVllVz61HRCCn87fzAVlcpvXpuLqnLv6YdQXhniT+8uIqRw8+g+FJdVUlJeydPfrkKB8w7vTkWl8vrsbFRh7MDOlFeG+HjhZipViQ/4WLetiLziMipCyjPfrkJRVKHCy0h/+3jZLr/bPW8t2GW9ZjvNif/4epf1ox74fJd1ERj4+48I+sUr5QhH/tX9fW8tKAOvnSc+4GfttkJ8IoybMJ2AX/CJsHTTTnw+4fZX55IY9LN2WxFtEiL/d986/7NM7PP54MCfwB0roHALzP4vZD0Dr1wG/jhI7giXvAqdDo52pDGlPm0dsaaq1BIfcCWWfp1SAUjxutyO7O1KMGlJbkyqM4d0q37uBws2AXDrmH4AzFy9DYD7znB3q6/wktGL144E6u59pKo8f81IKkIhLntqBgo8cenhVISUG5+fRUjhj2cNpKi0gvsmLySkyi1j+qHAI58uQ4Frj+1NcXklxeWVPD99DapwysAulFe6e1dAObZvOgBfLMlBgQFd2lBWEWJDXjEhVSpCIUorlMqQUlYZIlShTPthK8XlleQVlSFEvirVkoKJfcnpcOyv4KhfwrIP4e2bXbvD+COh00DXmyk5PeYbp2PF3kobsZ5EIkHE6w2Gr3rIko5t3AdwVfXakB5pALTzBkw8fXBXAF6YvgaAi4b/eGPtV0tdJ8p7T3dfWpZtdlPZ/t95g4Efk9NjF+/ay6w+XWUjzZKCaT78ARjwU7cU5MDCN93IrJsXuIH4nj4JDjkHDjmr0Yfwbu0amkgs0TRflhRM85TSEUZc75YJo6FoC5QVwod3uiW+DYy+xw2t0bbb3s9nmlxjJBZLNo3PkoJp/q4La+DLWQIvnAdFW+GDO9wSlwpJHeBnb0L73tGL00RcYySW1l7KkeZ2r9iwYcM0Kytr7wcak7vMDcT37T+gzDU40ulQOPgMGHAGZPSPueE1jKnS2ONJicgsVR221+MsKZhWYfsaWPyOSxLrvH7v7TKh78nQ7yQ44JgWc5OcMbWxpGBMXf4zBoq3QYc+7l6IihIQHySkwQn3Qr+ToU3XaEdpTKOqb1KwNgXT+lz76Y+Py4th1Tcw+RduiO93b3Xbg8ludNcz/w09RkJcUnRiNaaJWUnBmCqqkLMYln8E3zwMpTsBdTfLBRIgoS2c8Qh0PwLiU6MdrTENYtVHxuyv0gJYO91VMWU97bq8gqtqCiRCQhs48X7oPgza9bJGaxPTLCkY09hK8iF7pmuonvGkK0mom7+ApHTQkCtBnPkYdBvq5okwJkZYUjAm0kKVrrop+ztYN9PdYV1R7PaJ31U5xafCmN9Dt2GuYdvni27MptWypGBMNBRtg+wslyi+m+CqoKpKE+J3SeKIa1yVU7fD3Z3ZxjQBSwrGxIJQCLYsg/VZ8NkfXZVTRcmPicIf77rAdhsKXYdC1yGuQduYRhYTXVJFZCzwL9x0nE+p6gO1HHMBcB9ufoy5qlpzHmdjmi+fz00O1PEgN5McQFkRbJwLb93oksSmee6muir+eDhwNHQcAEvec1OUXvMJ+IPR+R1MqxKxkoKI+IFlwIlANjATGOdNwVl1TF/gFWC0qm4XkY6qmrOn81pJwbRIRdtgw2x491eul1NyBmxdDqEKt98XhPR+bnTYuCQY+1fIOMjdle3zRzV00zzEQklhOLBCVVd6AU0CzgQWhR1zLfCYqm4H2FtCMKbFSmoPfcbArfN+3FZR5oYDLy+Eg06DzQth6wooyoVJXoFafBBMhAFnQqdDvGUgpGRE5/cwzV4kk0I3YF3YejYwosYx/QBEZAquiuk+Vf2w5olE5DrgOoCePXvW3G1MyxSIg+u/3H17ST7kLoXcJfDFX6C8CH74DOa++OMxie1dSSKjP6z8ypUufjYZkjs0WfimeYpkUqjtTp6adVUBoC8wCugOfCMiA1U1b5cnqU4AJoCrPmr8UI1pRhLaQI8j3DL0sh+3F+TCf89yJYtex7nEsfBNKPH+nR7qDaldoLzEJYlRd0F6f8jo54b0MIbIJoVsoEfYendgQy3HTFfVcmCViCzFJYmZEYzLmJYpJQN+PmXXbaquHSJnIWxe5KqgFr0NO3e48Z6q+ILQYzik94UOfd3P9gdCuwOsgbuViWRSmAn0FZFewHrgIqBmz6K3gHHARBFJx1UnrYxgTMa0LiKQ2sktB452284e7268y1vrShMf3OkGBgxVwKLJbgTZ6uf7Ia0HFOe5oT2Ouhna93KTFbXLdO0ZpkWJWFJQ1QoRuRn4CNde8IyqLhSR+4EsVZ3s7TtJRBYBlcBvVHVrpGIyxnh8fu/DvRf0H7vrvsKtrkF72w+wbSVs/cENEli4Ez7+3a7HtunmEsSWZS5pjLkX0g6AtJ6uB5WNB9Xs2M1rxpj6UXXDi29bBdtXuYRRtayfDaHyXY8Xn6uKapfp7sUIJLj5Ktp0gzZdIKWza0w3TSIWuqQaY1oSEdd1Nqk9dD989/2lBbBjnauW+vAuqCh1jdjbV7t2Da2E167c9Tn+oJsitW03yJ7lhikffY9X5dUFUjvbMOVNzJKCMaZxxKe4u7A7DnBDd4RTdb2g8jdA/kbIXw9fP+TuxUho69o2Cja5kWbfuGbX5waTXXIozHV3ex92CbTtDm17uJ9turreU1ZV1Sis+sgYExtUoTQfdm6GnRvhvduhsgwOOtWtr/gMKkvdeFI1q6oCia5KqnCrq5IaPM6VNNp08UocXqkjEB+d3y0G2IB4xpiWKRSCwhzYsR52rP2x5JG/AVZ86hKH4n7W5Au40kbm0V7S6ArzXnXVVmc/DimdILkj+FteJYq1KRhjWiafz33rT+1ce9sG/Ngonr8Bdm6Cnd7PmU+7ZLFzE2z43lVJVZkwKuwaATf/RXKGmzPDH4RhV0FSB0hOd5MqJbV3d44ntW9R93JYUjDGtDzhjeKdB/64/fg7dj2uosy1ZRTkQMFmt3zzMFSWuxv4CrdAWYFb/+LPe7ie18U3pZNb1k53ieK4213JIznDJZPk9Jifkc+qj4wxpj4qyqBoKxRtccmieJsb3XbKI66No+dIl1x2bnI9rqrmzKgpmORuHvQHoeeRrvSx6mu3fsxtXjLrAB/d47Zd+UGjVGdZm4IxxkRTebGrnirIhbdvcqWNwy5xiWXey269fS/XOJ6f7Xpe1UogMc2NWZXaGW6Zs0/hWJuCMcZEUzDR3dmd1hNumr7rvpNrqYoqL3Ylj6oSyPu/cSWQQy9wpZOFb7kxqiLMSgrGGNMK1Lek4GuKYIwxxjQPlhSMMcZUs6RgjDGmmiUFY4wx1SwpGGOMqWZJwRhjTDVLCsYYY6pZUjDGGFPNkoIxxphqze6OZhHJBdbs49PTgS2NGE4kNZdYLc7G11xitTgbV6TjPEBVM/Z2ULNLCvtDRLLqc5t3LGgusVqcja+5xGpxNq5YidOqj4wxxlSzpGCMMaZaa0sKE6IdQAM0l1gtzsbXXGK1OBtXTMTZqtoUjDHG7FlrKykYY4zZA0sKxhhjqrWapCAiY0VkqYisEJG7oh1POBF5RkRyRGRB2Lb2IvKJiCz3fraLcow9ROQLEVksIgtF5JZYjNOLKUFEvhORuV6sf/C29xKRGV6sL4tIXLRjBRARv4h8LyLveusxF6eIrBaR+SIyR0SyvG2x+N6nichrIrLE+1s9Mkbj7O+9llVLvojcGguxtoqkICJ+4DHgFOBgYJyIHBzdqHYxERhbY9tdwGeq2hf4zFuPpgrg16o6ABgJ3OS9hrEWJ0ApMFpVBwNDgLEiMhJ4EPiHF+t24OooxhjuFmBx2HqsxvkTVR0S1pc+Ft/7fwEfqupBwGDc6xpzcarqUu+1HAIcDhQBbxILsapqi1+AI4GPwtbvBu6Odlw1YswEFoStLwW6eI+7AEujHWONeN8GTmwGcSYBs4ERuLtFA7X9TUQxvu64f/7RwLuAxGicq4H0Gtti6r0H2gCr8DrQxGqctcR9EjAlVmJtFSUFoBuwLmw929sWyzqp6kYA72fHKMdTTUQygcOAGcRonF6VzBwgB/gE+AHIU9UK75BY+Rv4J3AHEPLWOxCbcSrwsYjMEpHrvG2x9t73BnKBZ73quKdEJJnYi7Omi4CXvMdRj7W1JAWpZZv1xd0HIpICvA7cqqr50Y6nLqpaqa5o3h0YDgyo7bCmjWpXIvJTIEdVZ4VvruXQWPhbPVpVh+KqYG8SkeOiHVAtAsBQYLyqHgYUEgNVRXvitRedAbwa7ViqtJakkA30CFvvDmyIUiz1tVlEugB4P3OiHA8iEsQlhBdU9Q1vc8zFGU5V84Avce0gaSIS8HbFwt/A0cAZIrIamISrQvonsRcnqrrB+5mDq/seTuy999lAtqrO8NZfwyWJWIsz3CnAbFXd7K1HPdbWkhRmAn29Xh1xuOLa5CjHtDeTgcu9x5fj6vCjRkQEeBpYrKoPh+2KqTgBRCRDRNK8x4nAGFyD4xfAed5hUY9VVe9W1e6qmon7m/xcVS8hxuIUkWQRSa16jKsDX0CMvfequglYJyL9vU0nAIuIsThrGMePVUcQC7FGu5GlCRtzTgWW4eqWfxfteGrE9hKwESjHfdu5Gle3/Bmw3PvZPsoxHoOrxpgHzPGWU2MtTi/WQcD3XqwLgHu97b2B74AVuOJ6fLRjDYt5FPBuLMbpxTPXWxZW/f/E6Hs/BMjy3vu3gHaxGKcXaxKwFWgbti3qsdowF8YYY6q1luojY4wx9WBJwRhjTDVLCsYYY6pZUjDGGFPNkoIxxphqlhSMMcZUs6RgTD2IyBAROTVs/YzGGoLdGzI5qTHOZcz+svsUjKkHEbkCGKaqN0fg3Ku9c29pwHP8qlrZ2LEYYyUF06KISKY3ucp/vAl2PvaGuqjt2ANF5ENv5M9vROQgb/v5IrLAm6Tna29olPuBC70JUS4UkStE5FHv+IkiMl7cJEQrReR4cRMnLRaRiWHXGy8iWTUm/vkl0BX4QkS+8LaN8ya0WSAiD4Y9v0BE7heRGcCRIvKAiCwSkXki8rfIvKKm1Yn2rd622NKYC25eigpgiLf+CnBpHcd+BvT1Ho/AjT0EMB/o5j1O835eATwa9tzqddwkSZNwI5yeCeQDh+K+dM0Ki6W999OPG6RvkLe+Gm+uAlyCWAtk4Eb9/Bw4y9unwAVV58KNvS/hcdpiy/4uVlIwLdEqVZ3jPZ6FSxS78IYAPwp41Zt34UncpCYAU4CJInIt7gO8Pt5RVcUllM2qOl9VQ7ixgqquf4GIzMaNy3QIbhbAmo4AvlTVXHVzKrwAVA1TXYkbpRZc4ikBnhKRc3Azdxmz3wJ7P8SYZqc07HElUFv1kQ83mc2QmjtU9QYRGQGcBswRkd2O2cM1QzWuHwICItILuB04QlW3e9VKCbWcp7b5FKqUqNeOoKoVIjIcNxLoRcDNuKG3jdkvVlIwrZK6CYJWicj54IYGF5HB3uMDVXWGqt6LmxqzB7ATSN2PS7bBTfqyQ0Q64cbRrxJ+7hnA8SKS7s0tPg74qubJvJJOW1V9H7gVNzqoMfvNSgqmNbsEGC8i9wBBXLvAXOAhEemL+9b+mbdtLXCXV9X014ZeSFXnisj3uOqklbgqqioTgA9EZKOq/kRE7sbNqSDA+6pa25j6qcDbIpLgHXdbQ2MypjbWJdUYY0w1qz4yxhhTzaqPTIsnIo/h5kMO9y9VfTYa8RgTy6z6yBhjTDWrPjLGGFPNkoIxxphqlhSMMcZUs6RgjDGm2v8HVEmaUgT+3y4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#调整max_depth和min_child_weight之后再次调整n_estimators(6,6)\n",
    "xgb3 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=73,  #数值大没关系，cv会自动返回合适的n_estimators\n",
    "        max_depth=6,\n",
    "        min_child_weight=6,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "modelfit(xgb3, X_train, y_train, cv_folds = kfold)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "由于数据样本较少（为了及时完成作业），所有弱学习器的数目还是73"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'subsample': [0.3, 0.4, 0.5, 0.6, 0.7, 0.8], 'colsample_bytree': [0.6, 0.7, 0.8, 0.9]}\n"
     ]
    }
   ],
   "source": [
    "#以上已经调整得到的n_estimators最优值,max_depth=6,min_child_weight=6\n",
    "subsample = [i/10.0 for i in range(3,9)]\n",
    "colsample_bytree = [i/10.0 for i in range(6,10)]\n",
    "param_test4 = dict(subsample=subsample, colsample_bytree=colsample_bytree)\n",
    "print(param_test4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.71690, std: 0.04494, params: {'colsample_bytree': 0.6, 'subsample': 0.3},\n",
       "  mean: -0.71620, std: 0.04807, params: {'colsample_bytree': 0.6, 'subsample': 0.4},\n",
       "  mean: -0.71614, std: 0.03294, params: {'colsample_bytree': 0.6, 'subsample': 0.5},\n",
       "  mean: -0.72075, std: 0.04488, params: {'colsample_bytree': 0.6, 'subsample': 0.6},\n",
       "  mean: -0.72062, std: 0.03629, params: {'colsample_bytree': 0.6, 'subsample': 0.7},\n",
       "  mean: -0.71960, std: 0.04839, params: {'colsample_bytree': 0.6, 'subsample': 0.8},\n",
       "  mean: -0.71208, std: 0.04636, params: {'colsample_bytree': 0.7, 'subsample': 0.3},\n",
       "  mean: -0.71841, std: 0.04382, params: {'colsample_bytree': 0.7, 'subsample': 0.4},\n",
       "  mean: -0.72267, std: 0.03227, params: {'colsample_bytree': 0.7, 'subsample': 0.5},\n",
       "  mean: -0.72390, std: 0.04922, params: {'colsample_bytree': 0.7, 'subsample': 0.6},\n",
       "  mean: -0.72543, std: 0.04806, params: {'colsample_bytree': 0.7, 'subsample': 0.7},\n",
       "  mean: -0.72053, std: 0.05122, params: {'colsample_bytree': 0.7, 'subsample': 0.8},\n",
       "  mean: -0.70776, std: 0.03205, params: {'colsample_bytree': 0.8, 'subsample': 0.3},\n",
       "  mean: -0.71274, std: 0.04699, params: {'colsample_bytree': 0.8, 'subsample': 0.4},\n",
       "  mean: -0.72741, std: 0.04771, params: {'colsample_bytree': 0.8, 'subsample': 0.5},\n",
       "  mean: -0.72216, std: 0.04203, params: {'colsample_bytree': 0.8, 'subsample': 0.6},\n",
       "  mean: -0.71716, std: 0.04744, params: {'colsample_bytree': 0.8, 'subsample': 0.7},\n",
       "  mean: -0.73055, std: 0.04298, params: {'colsample_bytree': 0.8, 'subsample': 0.8},\n",
       "  mean: -0.71357, std: 0.03617, params: {'colsample_bytree': 0.9, 'subsample': 0.3},\n",
       "  mean: -0.71479, std: 0.03567, params: {'colsample_bytree': 0.9, 'subsample': 0.4},\n",
       "  mean: -0.72663, std: 0.04675, params: {'colsample_bytree': 0.9, 'subsample': 0.5},\n",
       "  mean: -0.72923, std: 0.04558, params: {'colsample_bytree': 0.9, 'subsample': 0.6},\n",
       "  mean: -0.72800, std: 0.04292, params: {'colsample_bytree': 0.9, 'subsample': 0.7},\n",
       "  mean: -0.73510, std: 0.05136, params: {'colsample_bytree': 0.9, 'subsample': 0.8}],\n",
       " {'colsample_bytree': 0.8, 'subsample': 0.3},\n",
       " -0.7077559710847329)"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb4 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=73,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=6,\n",
    "        min_child_weight=6,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch4 = GridSearchCV(xgb4, param_grid = param_test4, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch4.fit(X_train , y_train)\n",
    "\n",
    "gsearch4.grid_scores_, gsearch4.best_params_,     gsearch4.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 调整正则化参数：reg_alpha 和reg_lambda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_alpha': [1.5, 2], 'reg_lambda': [0.5, 1, 2]}"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg_alpha = [ 1.5, 2]    #default = 0, 测试0.1,1，1.5，2\n",
    "reg_lambda = [0.5, 1, 2]      #default = 1，测试0.1， 0.5， 1，2\n",
    "\n",
    "param_test5 = dict(reg_alpha=reg_alpha, reg_lambda=reg_lambda)\n",
    "param_test5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.70887, std: 0.03660, params: {'reg_alpha': 1.5, 'reg_lambda': 0.5},\n",
       "  mean: -0.71736, std: 0.03685, params: {'reg_alpha': 1.5, 'reg_lambda': 1},\n",
       "  mean: -0.70997, std: 0.03049, params: {'reg_alpha': 1.5, 'reg_lambda': 2},\n",
       "  mean: -0.70309, std: 0.03493, params: {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       "  mean: -0.71323, std: 0.03103, params: {'reg_alpha': 2, 'reg_lambda': 1},\n",
       "  mean: -0.71665, std: 0.03752, params: {'reg_alpha': 2, 'reg_lambda': 2}],\n",
       " {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       " -0.7030943914126934)"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb5 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=73,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=6,\n",
    "        min_child_weight=6,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch5 = GridSearchCV(xgb5, param_grid = param_test5, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch5.fit(X_train , y_train)\n",
    "\n",
    "gsearch5.grid_scores_, gsearch5.best_params_,     gsearch5.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 调用模型进行测试验证"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "#从测试数据中随机抽取一部分数据进行测试\n",
    "X_train_new,X_test_new,y_train_new,y_test_new = train_test_split(X_test, y_test, test_size = 0.02, random_state = 33)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(968, 227)"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_new.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_test6 = dict(reg_alpha=[2], reg_lambda=[0.5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb6 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=73,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=6,\n",
    "        min_child_weight=6,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=3, shuffle=True),\n",
       "       error_score='raise',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=0.7,\n",
       "       colsample_bytree=0.8, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=6, min_child_weight=6, missing=None, n_estimators=73,\n",
       "       n_jobs=1, nthread=None, objective='multi:softprob', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=3, silent=True,\n",
       "       subsample=0.3),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid={'reg_alpha': [2], 'reg_lambda': [0.5]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch6 = GridSearchCV(xgb6, param_grid = param_test6, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch6.fit(X_test , y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the model score is:  0.6033227771490383\n"
     ]
    }
   ],
   "source": [
    "print(\"the model score is: \", -gsearch6.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.603323 using {'reg_alpha': 2, 'reg_lambda': 0.5}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (-gsearch6.best_score_, gsearch6.best_params_))\n",
    "test_means = gsearch6.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch6.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch6.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch6.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch6.cv_results_).to_csv('my_preds_reg_alpha_reg_lambda.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
